
import warnings
warnings.filterwarnings('ignore')

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.preprocessing import OrdinalEncoder,OneHotEncoder,LabelEncoder
from sklearn.impute import KNNImputer


import xlrd
import numpy as np
import statistics
from sklearn.decomposition import PCA

from sklearn.preprocessing import MinMaxScaler,StandardScaler,RobustScaler
from bioinfokit.analys import get_data
from sklearn.manifold import TSNE
from bioinfokit.visuz import cluster

from sklearn.model_selection import train_test_split

from sklearn.metrics import accuracy_score

from sklearn.model_selection import GridSearchCV


# model evaluation
from sklearn import metrics
from sklearn.metrics import accuracy_score
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import mean_squared_error

# random forest
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
# gradient boosting
from sklearn.ensemble import GradientBoostingClassifier
# KNN
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from datetime import datetime
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.cluster import MeanShift
from sklearn.svm import SVR,SVC


import plotly.graph_objects as go
import plotly.io as pio
import plotly.express as px


############### importing countyData ###############
landarea_data = pd.read_csv("CountyData/5296US_landarea.csv")
pop_data = pd.read_csv("CountyData/5296US_pop.csv")
waterarea_data = pd.read_csv("CountyData/5296US_waterarea.csv")


############### importing EconomicData ###############
dowJonesIA_data = pd.read_csv("EconomicData/BCB-UDJIAD1.csv")
countyMHI_data = pd.read_csv("EconomicData/county1.csv") ## Median Household Income 
dowJonesIA_historical_data = pd.read_csv("EconomicData/Dow Jones Industrial Average Historical Data 2.csv")
currentDollarGDP_data = pd.read_excel("EconomicData/gdpcounty1218.xlsx","Current Dollar GDP",index_col=None)
currentDollarGDP_data.to_csv()
realGDP = pd.read_excel("EconomicData/gdpcounty1218.xlsx","Real GDP",index_col=None)
realGDP.to_csv()
realGDPGrowth = pd.read_excel("EconomicData/gdpcounty1218.xlsx","Real GDP Growth",index_col=None)
realGDPGrowth.to_csv()


############### importing GeneralDemographicData ###############
county_complete_data = pd.read_csv("GeneralDemographicData/county_complete.csv")


############### importing LifeExpectancyData ###############
lifeExpectancy_data = pd.read_excel("LifeExpectancyData/IHME_USA_LIFE_EXPECTANCY_1987_2007_Y2011M06D16.xlsx")
lifeExpectancy_data.to_csv()


############### importing USElectionResults data ###############
house_data = pd.read_csv("USElectionResults19762020/1976-2020-house.csv")
president_data = pd.read_csv("USElectionResults19762020/1976-2020-president.csv")
senate_data = pd.read_csv("USElectionResults19762020/1976-2020-senate.csv")


county_complete_data = pd.read_csv("GeneralDemographicData/county_complete.csv")


################################# filtering
wanted_states = ['California','Florida','South Dakota','Wyoming']


landarea_data.head()


landarea_data.isnull().sum()

FIPS         0
State        0
County       0
Land Area    0
dtype: int64


landarea_filtered = landarea_data.loc[landarea_data['State'].isin(wanted_states)]
landarea_filtered


cali_land = landarea_filtered[landarea_filtered['State'] == 'California']
wy_land = landarea_filtered[landarea_filtered['State'] == 'Wyoming']
fl_land = landarea_filtered[landarea_filtered['State'] == 'Florida']
sd_land = landarea_filtered[landarea_filtered['State'] == 'South Dakota']


my_dict = {'California': cali_land['Land Area'], 'Wyoming': wy_land['Land Area'],'Florida':fl_land['Land Area'], 'South Dakota': sd_land['Land Area']}
fig, ax = plt.subplots()
ax
ax.boxplot(my_dict.values())
ax.set_xticklabels(my_dict.keys())
plt.show()


out=[]
def iqr_outliers(df):
    q1 = df.quantile(0.25)
    q3 = df.quantile(0.75)
    iqr = q3-q1
    Lower_tail = q1 - 1.5 * iqr
    Upper_tail = q3 + 1.5 * iqr
    for i in df:
        if i > Upper_tail or i < Lower_tail:
            out.append(i)
    print("Outliers:",out)
    return out


cali_land.shape

(58, 4)


outliers = iqr_outliers(pd.Series(cali_land['Land Area'].unique()))
cali_land = cali_land[~cali_land['Land Area'].isin(outliers)]

Outliers: [26397.5, 21086.8, 51960.0]


cali_land.shape

(55, 4)


outliers = iqr_outliers(pd.Series(fl_land['Land Area']))
fl_land = fl_land[~fl_land['Land Area'].isin(outliers)]

Outliers: [26397.5, 21086.8, 51960.0, 5245.9, 5036.2, 5268.9, 4856.1]


outliers = iqr_outliers(pd.Series(wy_land['Land Area']))
wy_land = wy_land[~wy_land['Land Area'].isin(outliers)]

Outliers: [26397.5, 21086.8, 51960.0, 5245.9, 5036.2, 5268.9, 4856.1, 23783.1, 27003.0]


outliers = iqr_outliers(pd.Series(sd_land['Land Area']))
sd_land = sd_land[~sd_land['Land Area'].isin(outliers)]

Outliers: [26397.5, 21086.8, 51960.0, 5245.9, 5036.2, 5268.9, 4856.1, 23783.1, 27003.0, 8989.5, 7442.4]


land_area = pd.concat([cali_land, wy_land,fl_land,sd_land])


land_area.shape

(203, 4)


my_dict = {'California': cali_land['Land Area'], 'Wyoming': wy_land['Land Area'],'Florida':fl_land['Land Area'], 'South Dakota': sd_land['Land Area']}
fig, ax = plt.subplots()
ax
ax.boxplot(my_dict.values())
ax.set_xticklabels(my_dict.keys())
plt.show()


county_complete_data.head()


county_data_filtered = county_complete_data.loc[county_complete_data['state'].isin(wanted_states)]
county_data_filtered.dtypes

fips                         int64
state                       object
name                        object
pop2000                    float64
pop2010                      int64
                            ...   
uninsured_under_19_2019    float64
uninsured_under_6_2019     float64
veterans_2019              float64
white_2019                 float64
white_not_hispanic_2019    float64
Length: 188, dtype: object


county_cat = county_data_filtered.dtypes[county_data_filtered.dtypes == 'object']
county_cat

state                         object
name                          object
smoking_ban_2010              object
uninsured_age_under_6_2017    object
dtype: object


## converted this feature to float after identifying its null values
county_data_filtered['uninsured_age_under_6_2017'].replace('-',None,inplace=True)
county_data_filtered['uninsured_age_under_6_2017'].replace(np.nan,None,inplace=True)


county_data_filtered['uninsured_age_under_6_2017'] = county_data_filtered['uninsured_age_under_6_2017'].astype(float)
county_cat = county_data_filtered.dtypes[county_data_filtered.dtypes == 'object']
county_cat

state               object
name                object
smoking_ban_2010    object
dtype: object


county_nulls = county_data_filtered.isnull().sum()
county_nulls[county_nulls > 0]

age_under_5_2017              1
age_over_65_2017              1
median_age_2017               1
black_2010                    4
black_2017                    1
                           ... 
unemployment_rate_2017        1
mean_work_travel_2019        98
poverty_2019                115
poverty_65_and_over_2019    115
poverty_under_18_2019       115
Length: 100, dtype: int64


county_data_filtered.shape

(214, 188)


limitPer = county_data_filtered.shape[0] * .50
county_data_filtered.dropna(thresh=limitPer, axis=1, inplace=True)


county_data_filtered.shape

(214, 180)


## one hot encoding for this feature
oe_smoking = OrdinalEncoder(categories=[['none', 'partial','comprehensive']])
county_data_filtered['smoking_ban_2010'] = oe_smoking.fit_transform(county_data_filtered[['smoking_ban_2010']])


county_data_filtered['smoking_ban_2010'] = county_data_filtered['smoking_ban_2010'].astype('category')


county_nulls = county_data_filtered.isnull().sum()
county_nulls[county_nulls > 0]

age_under_5_2017              1
age_over_65_2017              1
median_age_2017               1
black_2010                    4
black_2017                    1
                             ..
civilian_labor_force_2017     1
employed_2017                 1
unemployed_2017               1
unemployment_rate_2017        1
mean_work_travel_2019        98
Length: 92, dtype: int64


county_complete_numericals = county_data_filtered.iloc[:,3:]
county_complete_numericals = county_complete_numericals.select_dtypes(include='number')


plt.figure(figsize=(250, 250))
sns.heatmap(county_complete_numericals.corr(), annot=True, cmap='mako_r')
plt.show()


#dropping highly correlated features
cor_matrix = county_complete_numericals.corr().abs()
lower_tri = cor_matrix.where(np.tril(np.ones(cor_matrix.shape),-1).astype(np.bool))
to_drop = [column for column in lower_tri.columns if any(lower_tri[column] > 0.95)]


print("Before",county_data_filtered.shape)
county_data_filtered.drop(to_drop, axis=1,inplace=True)
print("After",county_data_filtered.shape)

Before (214, 180)
After (214, 81)


lifeExpectancy_data.head()


lifeExpectancy_filtered = lifeExpectancy_data.loc[lifeExpectancy_data['State'].isin(list(map(str.upper,wanted_states)))]
lifeExpectancy_filtered


## replacing empty spaces with Null
lifeExpectancy_filtered.replace(" ",None)
lifeExpectancy_filtered.isnull().sum()

fips                                                               0
State                                                              0
County                                                             0
Year                                                               0
Male life expectancy (years)                                       0
Years behind international frontier (male)                         0
Female life expectancy (years)                                     0
Years behind international frontier (female)                       0
White male life expectancy (years)                                 0
White female life expectancy (years)                               0
Black male life expectancy (years)                              3392
Black female life expectancy (years)                            3322
Closest ranked countries for male life expectancy (higher)        20
Closest ranked countries for female life expectancy (higher)       0
Closest ranked countries for male life expectancy (lower)          0
Closest ranked countries for female life expectancy (lower)        0
Rank (male)                                                        0
Rank (female)                                                      0
Male life expectancy change 1987 to 2007 (years)                   0
Female life expectancy change 1987 to 2007 (years)                 0
Male life expectancy change 1987 to 1997 (years)                   0
Female life expectancy change 1987 to 1997 (years)                 0
Male life expectancy change 1997 to 2007 (years)                   0
Female life expectancy change 1997 to 2007 (years)                 0
dtype: int64


lifeExpectancy_filtered.dtypes

fips                                                              int64
State                                                            object
County                                                           object
Year                                                              int64
Male life expectancy (years)                                    float64
Years behind international frontier (male)                       object
Female life expectancy (years)                                  float64
Years behind international frontier (female)                     object
White male life expectancy (years)                              float64
White female life expectancy (years)                            float64
Black male life expectancy (years)                              float64
Black female life expectancy (years)                            float64
Closest ranked countries for male life expectancy (higher)       object
Closest ranked countries for female life expectancy (higher)     object
Closest ranked countries for male life expectancy (lower)        object
Closest ranked countries for female life expectancy (lower)      object
Rank (male)                                                       int64
Rank (female)                                                     int64
Male life expectancy change 1987 to 2007 (years)                float64
Female life expectancy change 1987 to 2007 (years)              float64
Male life expectancy change 1987 to 1997 (years)                float64
Female life expectancy change 1987 to 1997 (years)              float64
Male life expectancy change 1997 to 2007 (years)                float64
Female life expectancy change 1997 to 2007 (years)              float64
dtype: object


lifeExpectancy_filtered.replace("50+",51,inplace=True)
lifeExpectancy_filtered.dtypes

fips                                                              int64
State                                                            object
County                                                           object
Year                                                              int64
Male life expectancy (years)                                    float64
Years behind international frontier (male)                        int64
Female life expectancy (years)                                  float64
Years behind international frontier (female)                      int64
White male life expectancy (years)                              float64
White female life expectancy (years)                            float64
Black male life expectancy (years)                              float64
Black female life expectancy (years)                            float64
Closest ranked countries for male life expectancy (higher)       object
Closest ranked countries for female life expectancy (higher)     object
Closest ranked countries for male life expectancy (lower)        object
Closest ranked countries for female life expectancy (lower)      object
Rank (male)                                                       int64
Rank (female)                                                     int64
Male life expectancy change 1987 to 2007 (years)                float64
Female life expectancy change 1987 to 2007 (years)              float64
Male life expectancy change 1987 to 1997 (years)                float64
Female life expectancy change 1987 to 1997 (years)              float64
Male life expectancy change 1997 to 2007 (years)                float64
Female life expectancy change 1997 to 2007 (years)              float64
dtype: object


lifeExpec_numeric = lifeExpectancy_filtered.select_dtypes('number')
lifeExpec_numeric = lifeExpec_numeric.iloc[:,2:]


plt.figure(figsize=(20, 20))
sns.heatmap(lifeExpec_numeric.corr(), annot=True, cmap='mako_r')
plt.show()


limitPer = lifeExpectancy_filtered.shape[0] * .50
lifeExpectancy_filtered.dropna(thresh=limitPer, axis=1, inplace=True)


lifeExpectancy_filtered


lifeExpectancy_filtered=lifeExpectancy_filtered.groupby("fips").mean()
lifeExpectancy_filtered=lifeExpectancy_filtered.reset_index()


lifeExpectancy_filtered


land_area.rename(str.lower, axis='columns',inplace=True)
lifeExpectancy_filtered.rename(str.lower, axis='columns',inplace=True)
county_data_filtered.rename(str.lower, axis='columns',inplace=True)


lifeExpectancy_data['State'] = lifeExpectancy_data['State'].str.title()


b_data = pd.merge(lifeExpectancy_filtered,land_area, on='fips')
b_data = pd.merge(b_data,county_data_filtered, on='fips')


b_data.drop(['county', 'state_y','name'],axis=1, inplace=True)


b_data.rename({'state_x' : 'state'},inplace=True,axis=1)


b_nulls = b_data.isnull().sum()
b_nulls[b_nulls > 0]

pac_isl_2010                                      55
percent_change_private_nonfarm_employment_2009     3
women_owned_firms_2007                            68
manufacturer_shipments_2007                       16
sales_per_capita_2007                              4
mean_work_travel_2019                             94
dtype: int64


%%time
# use KNN to impute missing values
imputer = KNNImputer(n_neighbors=5, weights="distance")
imputed = imputer.fit_transform(b_data.drop(['fips','state'],axis=1))

# replace values in the dataset with new imputed values
b_data[b_data.columns.drop(['fips','state'])] = imputed

Wall time: 34 ms


b_data_num = b_data.select_dtypes('number')


fts = ['land area','rank (female)','pop_2019','native_2019',
       'median_household_income_2019','sales_per_capita_2007','white_not_hispanic_2019','fips','persons_per_household_2010','foreign_born_2010','hs_grad_2019']
all_fts = fts.copy()

all_fts.append('state')

pca_fts = b_data[b_data.columns[b_data.columns.isin(all_fts)]]
b_cpy = b_data.copy()

scaler = StandardScaler()
x = b_cpy.loc[:,fts]
y = b_cpy.loc[:,['state']].values

x = scaler.fit_transform(x)




pca = PCA(n_components=2)
pcs = pca.fit(x)
pcs.explained_variance_ratio_
loadings = pcs.components_
num_pc = pcs.n_features_
pc_list = ["PC"+str(i) for i in list(range(1, num_pc+1))]
loadings_df = pd.DataFrame.from_dict(dict(zip(pc_list, loadings)))
loadings_df['variable'] = fts
loadings_df = loadings_df.set_index('variable')
loadings_df
show = loadings_df.loc[:,['PC1','PC2']]
show['PC2'].sort_values(ascending=False).head(20)

variable
median_household_income_2019    0.527836
foreign_born_2010               0.411353
sales_per_capita_2007           0.374233
pop_2019                        0.355486
hs_grad_2019                    0.191809
land area                       0.094885
white_not_hispanic_2019        -0.064748
persons_per_household_2010     -0.126964
fips                           -0.177927
rank (female)                  -0.299887
native_2019                    -0.313146
Name: PC2, dtype: float64


pca_scores = pca.fit_transform(x)


# component loadings
loadings = pcs.components_
loadings


# get eigenvalues (variance explained by each PC)  
pcs.explained_variance_
target = b_cpy['state']
colors = ['#1B2430','#FFC54D','#F15412','#53BF9D']

# get biplot
pca_scores = pca.fit_transform(x)
cluster.biplot(cscore=pca_scores, loadings=loadings, labels=fts, var1=round(pcs.explained_variance_ratio_[0]*100, 2),
    var2=round(pcs.explained_variance_ratio_[1]*100, 2), colorlist=target,show=True,dim=(10,10),dotsize=(15),arrowlinewidth=2,colordot=colors)


from sklearn.manifold import TSNE

x_t = b_cpy.loc[:,fts]
# Separating out the target
y = b_cpy.loc[:,['state']].values
x_t = scaler.fit_transform(x_t)


tsne_em = TSNE(n_components=2, perplexity=30.0,
               n_iter=2500, verbose=1).fit_transform(x)
color_class = b_cpy['state'].to_numpy()
cluster.tsneplot(score=tsne_em, colorlist=color_class, legendpos='upper right', legendanchor=(1.15, 1),show=True,
                 dim=(7,7),colordot=10)

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.001s...
[t-SNE] Computed neighbors for 202 samples in 0.005s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.290343
[t-SNE] KL divergence after 250 iterations with early exaggeration: 58.183624
[t-SNE] KL divergence after 1000 iterations: 0.289460


def create_pca(x,fts):
    pca_scores = pca.fit_transform(x)


    # component loadings
    loadings = pcs.components_
    loadings


    # get eigenvalues (variance explained by each PC)  
    pcs.explained_variance_
    target = b_cpy['state']
    colors = ['#1B2430','#FFC54D','#F15412','#53BF9D']

    # get biplot
    pca_scores = pca.fit_transform(x)
    cluster.biplot(cscore=pca_scores, loadings=loadings, labels=fts, var1=round(pcs.explained_variance_ratio_[0]*100, 2),
        var2=round(pcs.explained_variance_ratio_[1]*100, 2), colorlist=target,show=True,dim=(10,10),dotsize=(15),arrowlinewidth=2,colordot=colors)
    
    

    
def create_tsne(fts):
    from sklearn.manifold import TSNE

    x_t = b_cpy.loc[:,fts]
    # Separating out the target
    y = b_cpy.loc[:,['state']].values
    x_t = scaler.fit_transform(x_t)


    tsne_em = TSNE(n_components=2, perplexity=30.0,
                   n_iter=2500, verbose=1).fit_transform(x)
    color_class = b_cpy['state'].to_numpy()
    cluster.tsneplot(score=tsne_em, colorlist=color_class, legendpos='upper right', legendanchor=(1.15, 1),show=True,
                     dim=(7,7),colordot=10)


for i in fts:
    try_fts = fts.copy()
    try_fts.remove(i)
    print("Removed",i)
    x = b_cpy.loc[:,try_fts]
    y = b_cpy.loc[:,['state']].values
    x = scaler.fit_transform(x)
    create_pca(x,try_fts)
    create_tsne(try_fts)

Removed land area

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.000s...
[t-SNE] Computed neighbors for 202 samples in 0.004s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.148835
[t-SNE] KL divergence after 250 iterations with early exaggeration: 56.787716
[t-SNE] KL divergence after 1150 iterations: 0.305002

Removed rank (female)

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.000s...
[t-SNE] Computed neighbors for 202 samples in 0.004s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.169926
[t-SNE] KL divergence after 250 iterations with early exaggeration: 56.796196
[t-SNE] KL divergence after 1250 iterations: 0.284774

Removed pop_2019

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.000s...
[t-SNE] Computed neighbors for 202 samples in 0.005s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.279339
[t-SNE] KL divergence after 250 iterations with early exaggeration: 57.313396
[t-SNE] KL divergence after 1000 iterations: 0.277537

Removed native_2019

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.001s...
[t-SNE] Computed neighbors for 202 samples in 0.005s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.261668
[t-SNE] KL divergence after 250 iterations with early exaggeration: 58.187778
[t-SNE] KL divergence after 850 iterations: 0.267058

Removed median_household_income_2019

[t-SNE] Computing 91 nearest neighbors...
[t-SNE] Indexed 202 samples in 0.000s...
[t-SNE] Computed neighbors for 202 samples in 0.005s...
[t-SNE] Computed conditional probabilities for sample 202 / 202
[t-SNE] Mean sigma: 1.218048
[t-SNE] KL divergence after 250 iterations with early exaggeration: 57.665703
[t-SNE] KL divergence after 1000 iterations: 0.239819


############### importing GeneralDemographicData ###############
county_complete = pd.read_csv("GeneralDemographicData/county_complete.csv")
house_data = pd.read_csv("USElectionResults19762020/1976-2020-house.csv")


house_filtered = house_data.loc[((house_data.year <=2012) & (house_data.year >=2002))]
house_filtered


#summing candidatevotes in each state in a specific election cycle (year)
house_filtered['totalvotes_state/year'] = house_filtered.groupby(['state_fips','year'])['candidatevotes'].transform(sum)
house_filtered = house_filtered[['year','state','state_fips','totalvotes_state/year']]
house_filtered.drop_duplicates(inplace=True)
house_filtered.reset_index(inplace=True,drop=True)
house_filtered


pops = []
for i in county_complete.columns:
    if 'pop' in i:
        pops.append(i)

pops

['pop2000',
 'pop2010',
 'pop2011',
 'pop2012',
 'pop2013',
 'pop2014',
 'pop2015',
 'pop2016',
 'pop2017',
 'pop_2019']


## summing population in each state
cc_pops = county_complete.loc[:,(['fips','state','name'] + pops)]
cc_pops

cc_allpops = cc_pops.groupby(['state']).sum()
cc_allpops = cc_allpops.iloc[:,:5]
cc_allpops.drop("fips",inplace=True,axis=1)
cc_allpops


for i in cc_allpops.columns:
    if 'pop' in i:
        cc_allpops.rename({i: i.replace('pop', '')},inplace=True, axis=1)
cc_allpops.head()


exst_pops = list(map(int, cc_allpops.columns))
exst_pops

[2000, 2010, 2011, 2012]


miss_pops = []
for i in house_filtered.year.unique():
    if i not in exst_pops:
        miss_pops.append(i)
miss_pops

cc_allpops.reset_index(inplace=True)


def estimated_pop (curr_year,year1,year2,P1,P2):
    n = (curr_year - year1)*12
    N = (year2-year1)*12
    P_est = (n/N)*(P2-P1)
    P_est += P1
    return int(P_est)

# v = estimated_pop(2002,2000,2010,4447100.0,4779736)
# v


for i in miss_pops:
    cc_allpops[i] = cc_allpops.apply(lambda row : (estimated_pop(i, 2000,2010,row['2000'],row['2010'])),axis=1)

cc_allpops.head()


house_filtered['state'] = house_filtered['state'].str.lower()
house_filtered['state'] = house_filtered['state'].str.title() #capitlize first letter for each word in a string, eg: new york --> New York
house_filtered.head()


l = house_filtered.state.unique()
for i in cc_allpops.state.unique():
    if i not in l:
        print(i)

District of Columbia


def drop_DC(df):
    if('District of Columbia' in df.state.unique()):
        df = df.drop(df[df.state == 'District of Columbia'].index)
    elif ('District Of Columbia' in df.state.unique()):
        df = df.drop(df[df.state == 'District Of Columbia'].index)
    return df


cc_allpops = drop_DC(cc_allpops)


pop_age18 = county_complete.loc[:,(['fips','state','name','age_under_18_2010','age_over_18_2019','pop2010','pop_2019'])]
pop_age18.isnull().sum()

fips                 0
state                0
name                 0
age_under_18_2010    0
age_over_18_2019     0
pop2010              0
pop_2019             0
dtype: int64


pop_age18


# we will calculate the only two features that we have regarding age under/over 18
pop_age18['above18_2010'] = pop_age18.apply(lambda row : int((((100-row['age_under_18_2010'])/100)*row['pop2010'])),axis=1)
pop_age18['above18_2019'] = pop_age18.apply(lambda row : int((((row['age_over_18_2019'])/100)*row['pop_2019'])),axis=1)

pop_age18.drop(['age_under_18_2010','age_over_18_2019','pop2010','pop_2019','fips'],inplace=True,axis=1)
pop_age18.head()


## data is grouped by state 
pop_age18 = pop_age18.groupby(['state']).sum()
pop_age18.reset_index(inplace=True)
pop_age18.head()


def estimated_growth_rate(P1,P2,t):
    gr = ((P2-P1)/P2)/t
    return gr


def estimated_pop_based_on_gr(curr_year,pop1,pop2,year1,year2):
    gr = estimated_growth_rate(pop1,pop2,abs(year2-year1))
    n = abs(curr_year - year1)
    if(curr_year > year1):
        P_est = (pow((1+gr),n))*pop1
    else:
        P_est = pop1/(pow((1+gr),n))
    return int(P_est)


miss_pops18 = []
for i in house_filtered.year.unique():
    if str(i) not in str(pop_age18.columns):
        miss_pops18.append(i)
miss_pops18

[2002, 2004, 2006, 2008, 2012]


for i in miss_pops18:
    pop_age18["above18_"+str(i)] = pop_age18.apply(lambda row : (estimated_pop_based_on_gr(i,row['above18_2010'],row['above18_2019'],2010,2019)),axis=1)

pop_age18.head()


pop_age18 = drop_DC(pop_age18)


## joining data to calculate label
c_data = pd.merge(house_filtered,pop_age18,on='state')
c_data


def calc_voter_turnout(total_votes,reg_voters):
    turnout = round((total_votes/reg_voters),3)
    return turnout


c_data['voter_turnout'] = c_data.apply(lambda row : (calc_voter_turnout(row['totalvotes_state/year'],
                                                                  row['above18_'+str(row.year)])),axis=1)

c_data.head()


state_abv = house_data.loc[:,['state','state_po']]
state_abv['state'] = state_abv['state'].str.title()
state_abv.drop_duplicates(inplace=True)
state_abv = drop_DC(state_abv)
state_abv.reset_index(inplace=True,drop=True)


def turnout_rates_graph(df,year,color):
    
    df['text'] = df.apply(lambda x: x['state'] +'<br>' + 'Turnout Rate: ' + format(x['voter_turnout']*100)+'%' + 
                          '<br>'+ 'Total Ballots Counted: '+str(x['totalvotes_state/year']) + 
                          '<br>'+ 'Voting-Eligible Population: '+str(x['above18_'+str(year)]),axis=1)


    fig2 = go.Figure(data=go.Choropleth(
        locations=state_abv['state_po'],
        z=df['voter_turnout'],
        locationmode='USA-states',
        colorscale= color,
        autocolorscale=False,
        text=df['text'], # hover text
        marker_line_color='white', # line markers between states
        colorbar_title="Voter Turnout Rates - "+str(year),
        hoverinfo="text"
    ))

    fig2.update_layout(
        title_text='',
        geo = dict(
            scope='usa',
            projection=go.layout.geo.Projection(type = 'albers usa'),
            showlakes=True, # lakes
            lakecolor='rgb(255, 255, 255)')
    )


    fig2.show()


colors = ['teal','darkmint','burg','emrld','tempo','matter']
c = 0
for i in range(2002,2014,2):
    df = c_data.loc[c_data['year'] == i]
    turnout_rates_graph(df,i,colors[c])
    c+=1


for i in county_complete.columns:
    if 'women' in i or 'female' in i or 'woman' in i:
        print(i)

female_2010
women_16_to_50_birth_rate_2017
women_owned_firms_2007


#Horizontal Bar Plot
states = county_complete.state.unique()
#We create 6 empty list to keep each ethnicity
share_white = []
share_black = []
share_native_american = []
share_asian = []
share_hispanic = []
share_pac_isl = []

#Find the number of each ethnicity in the States
for i in states:
    x = county_complete[county_complete['state']==i]
    share_white.append(sum(x.white_2019)/len(x))
    share_black.append(sum(x.black_2019) / len(x))
    share_native_american.append(sum(x.native_2019) / len(x))
    share_asian.append(sum(x.asian_2019) / len(x))
    share_hispanic.append(sum(x.hispanic_2019) / len(x))
    share_pac_isl.append(sum(x.pac_isl_2019) / len(x)) 

# visualization
f,ax = plt.subplots(figsize = (9,15))
sns.barplot(x=share_white,y=states,color='green',alpha = 0.5,label='White' )
sns.barplot(x=share_black,y=states,color='blue',alpha = 0.7,label='Black')
sns.barplot(x=share_native_american,y=states,color='cyan',alpha = 0.6,label='Native American')
sns.barplot(x=share_asian,y=states,color='yellow',alpha = 0.6,label='Asian')
sns.barplot(x=share_hispanic,y=states,color='red',alpha = 0.6,label='Hispanic')
sns.barplot(x=share_hispanic,y=states,color='grey',alpha = 0.6,label='Hawaii or Pacific Islander ')

ax.legend(loc='lower right',frameon = True)     
ax.set(xlabel='Percentage of Races', ylabel='States',title = "Percentage of State's Population According to Ethnics - based on 2019 data ")

[Text(0.5, 0, 'Percentage of Races'),
 Text(0, 0.5, 'States'),
 Text(0.5, 1.0, "Percentage of State's Population According to Ethnics - based on 2019 data ")]


ethnic = county_complete.loc[:,['state','name','white_2010','white_2019','black_2010','black_2019','hispanic_2010'
                                ,'hispanic_2019','pop2010','pop_2019']]
ethnic.isnull().sum()

state             0
name              0
white_2010        0
white_2019        0
black_2010       17
black_2019        0
hispanic_2010     0
hispanic_2019     0
pop2010           0
pop_2019          0
dtype: int64


# use KNN to impute missing values
imputer = KNNImputer(n_neighbors=5, weights="distance")
imputed = imputer.fit_transform(ethnic.drop(['state','name'],axis=1))

# replace values in the dataset with new imputed values
ethnic[ethnic.columns.drop(['name','state'])] = imputed


ethnic.rename(columns={'pop2010':'pop_2010'},inplace=True)


for i in [2010,2019]:
    for j in ['white_','black_','hispanic_']:
        index = j+str(i)
        ethnic['pop_'+index] =  ethnic.apply(lambda row : int((row[index]/100)*row['pop_'+str(i)]),axis=1)


ethnic.head()


ethnic.drop(['white_2010','white_2019','black_2010','black_2019','hispanic_2010','hispanic_2019'],inplace=True,axis=1)


ethnic_grouped = ethnic.groupby(['state']).sum()
ethnic_grouped.head()


ethnic_grouped.reset_index(inplace=True)


for i in miss_pops18:
    for j in ['white_','black_','hispanic_']:
        index = 'pop_'+j
        ethnic_grouped[index+str(i)] = ethnic_grouped.apply(lambda row : (estimated_pop_based_on_gr(i,row[index+'2010'],row[index+'2019'],2010,2019)),axis=1)
        

ethnic_grouped.head()


ethnic_grouped = drop_DC(ethnic_grouped)
cc_allpops.drop(['2011','2000'],inplace=True,axis=1)
ethnic_grouped = pd.merge(ethnic_grouped,cc_allpops,on='state')
ethnic_grouped.head()


ethnic_grouped.drop(['pop_2010','pop_2019'],inplace=True,axis=1)
ethnic_grouped.columns = ethnic_grouped.columns.astype(str)


for i in range(2002,2014,2):
    for j in ['pop_white_','pop_black_','pop_hispanic_']:
        index = j+str(i)
        ethnic_grouped['percent_'+index] =  ethnic_grouped.apply(lambda row : round((row[index]/row[str(i)])*100,2),axis=1)


ethnic_grouped.head()


percentages = []
for i in ethnic_grouped.columns:
    if 'percent_pop' in i:
        percentages.append(i)


ethnic_grouped = ethnic_grouped.loc[:,(['state']+percentages)]
ethnic_grouped = pd.merge(ethnic_grouped,pop_age18,on='state')
ethnic_grouped.head()


for i in range(2002,2014,2):
    for j in ['white_','black_','hispanic_']:
        index = j+str(i)
        ethnic_grouped['above18_'+index] =  ethnic_grouped.apply(lambda row : int((row['percent_pop_'+index]/100)*row['above18_'+str(i)]),axis=1)


##group has above18, above18_ethn, ethn_percent
ethnic_grouped.head()


for i in county_complete.columns:
    if 'income' in i or 'employ' in i or 'bachelors' in i or 'hs_grad' in i or 'poverty' in i or 'college' in i:
        print(i)

hs_grad_2010
hs_grad_2016
hs_grad_2017
some_college_2016
some_college_2017
bachelors_2010
bachelors_2016
bachelors_2017
per_capita_income_2010
per_capita_income_2017
median_household_income_2010
median_household_income_2016
median_household_income_2017
private_nonfarm_employment_2009
percent_change_private_nonfarm_employment_2009
nonemployment_establishments_2009
poverty_2010
poverty_2016
poverty_2017
poverty_age_under_5_2017
poverty_age_under_18_2017
employed_2007
unemployed_2007
unemployment_rate_2007
employed_2008
unemployed_2008
unemployment_rate_2008
employed_2009
unemployed_2009
unemployment_rate_2009
employed_2010
unemployed_2010
unemployment_rate_2010
employed_2011
unemployed_2011
unemployment_rate_2011
employed_2012
unemployed_2012
unemployment_rate_2012
employed_2013
unemployed_2013
unemployment_rate_2013
employed_2014
unemployed_2014
unemployment_rate_2014
employed_2015
unemployed_2015
unemployment_rate_2015
employed_2016
unemployed_2016
unemployment_rate_2016
employed_2017
unemployed_2017
unemployment_rate_2017
bachelors_2019
hs_grad_2019
mean_household_income_2019
median_household_income_2019
median_individual_income_2019
median_individual_income_age_25plus_2019
per_capita_income_2019
poverty_2019
poverty_65_and_over_2019
poverty_under_18_2019
unemployment_rate_2019


so_ec2 = ['hs_grad_2017','some_college_2017','bachelors_2017',
          'median_household_income_2017','poverty_2017','employed_2017','unemployed_2017',
         'unemployment_rate_2017','mean_household_income_2019','per_capita_income_2019']
so_economic2 = county_complete.loc[:,(so_ec2)]


plt.figure(figsize=(20, 20))
sns.heatmap(so_economic2.corr(), annot=True, cmap='mako_r')
plt.show()


cor_matrix = so_economic2.corr().abs()
lower_tri = cor_matrix.where(np.tril(np.ones(cor_matrix.shape),-1).astype(np.bool))
to_drop2 = [column for column in lower_tri.columns if any(lower_tri[column] >= 0.90)]
to_drop2

['median_household_income_2017', 'employed_2017', 'mean_household_income_2019']


so_ec_fts = ['bachelors_2019','bachelors_2010','unemployed_2010','unemployed_2017',
'poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019','per_capita_income_2010',
'per_capita_income_2019']


so_economic = county_complete.loc[:,(['state' , 'name','pop2010','pop2017','pop_2019']+so_ec_fts)]
print(so_economic.shape)
so_economic.head()

(3142, 15)


so_economic.isnull().sum()

state                     0
name                      0
pop2010                   0
pop2017                   3
pop_2019                  0
bachelors_2019            0
bachelors_2010            0
unemployed_2010           3
unemployed_2017           3
poverty_2017              2
poverty_2010              0
hs_grad_2010              0
hs_grad_2019              0
per_capita_income_2010    0
per_capita_income_2019    0
dtype: int64


imputer = KNNImputer(n_neighbors=5, weights="distance")
imputed = imputer.fit_transform(so_economic.drop(['state','name'],axis=1))

# replace values in the dataset with new imputed values
so_economic[so_economic.columns.drop(['name','state'])] = imputed

so_economic.rename(columns={'pop_2019':'pop2019'},inplace=True)


#make normalization
states = county_complete.state.unique()
plot1 = pd.DataFrame(so_economic.groupby(['state'])['per_capita_income_2010'].sum())
plot2 = pd.DataFrame(so_economic.groupby(['state'])['poverty_2010'].sum())

plot1['state_income_ratio'] = plot1['per_capita_income_2010']/max( plot1['per_capita_income_2010'])
plot2['state_poverty_ratio'] = plot2['poverty_2010']/max( plot2['poverty_2010'])
data = pd.concat([plot1,plot2['state_poverty_ratio']],axis=1)
data.sort_values('per_capita_income_2010',inplace=True)

# visualize
f,ax1 = plt.subplots(figsize =(30,10))
sns.pointplot(x=states,y='state_income_ratio',data=data,color='#1b9e77',alpha=0.8)
sns.pointplot(x=states,y='state_poverty_ratio',data=data,color='#d95f02',alpha=0.8)
plt.text(2,0.9,'Income Ratio',color='#1b9e77',fontsize = 20,style = 'italic')
plt.text(2,0.85,'Poverty',color='#d95f02',fontsize = 20,style = 'italic')
plt.xlabel('States',fontsize = 20,color='black')
plt.xticks(rotation= 90)
plt.ylabel('Values',fontsize = 20,color='black')
plt.title('Income Rate  VS  Poverty Rate in 2010',fontsize = 20,color='black')
plt.grid()


## converting these features values from percentages to numeric values
convert_vals = ['bachelors_2019','bachelors_2010','poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019']
for i in convert_vals:
    index = i.split('_')
    so_economic[i] = so_economic.apply(lambda row : int((row[i]/100)*row['pop'+index[-1]]),axis=1)


socio_econ_grouped = so_economic.groupby('state').sum()
socio_econ_grouped.head()
socio_econ_grouped.reset_index(inplace=True)
socio_econ_grouped = drop_DC(socio_econ_grouped)
socio_econ_grouped.drop(['pop2019','pop2017','pop2010'],inplace=True,axis=1)


cols = socio_econ_grouped.columns
fts = ['bachelors','unemployed','poverty','hs_grad','per_capita_income']
for i in range(2002,2014,2):
    for j in fts:
        ft = j+'_'
        if((ft+str(i)) not in cols):
            if ft+'2010' in cols:
                pop1 = ft+'2010'
                y1 = 2010
                if ft+'2017' in cols:
                    pop2 = ft+'2017'
                    y2=2017
                else:
                    pop2 = ft+'2019'
                    y2 = 2019
            socio_econ_grouped[ft+str(i)] = socio_econ_grouped.apply(lambda row : (estimated_pop_based_on_gr(i,row[pop1],row[pop2],y1,y2)),axis=1)


socio_econ_grouped.drop(['bachelors_2019','unemployed_2017','poverty_2017','hs_grad_2019',
                        'per_capita_income_2019',],inplace=True,axis=1)
socio_econ_grouped.reset_index(inplace=True,drop=True)
socio_econ_grouped.head()


final = pd.DataFrame(columns=['year','state']+fts+['white','black','hispanic'])
row_data = 0
index=0
for i in range(2002,2014,2):
    row_data=0
    new_row = []
    new_row.append(i)
    for j in socio_econ_grouped.state.unique():
        new_row.append(j)
        for ft in fts:
            val= socio_econ_grouped.loc[row_data,[ft+"_"+str(i)]]
            new_row.append(val[0])
        for ft in ['white','black','hispanic']:
            val= ethnic_grouped.loc[row_data,['above18_'+ft+"_"+str(i)]]
            new_row.append(val[0])             
        final.loc[index] = new_row
        new_row = []
        new_row.append(i)
        index+=1
        row_data+=1
final


voter = c_data.loc[:,['state','year','voter_turnout']]
final = pd.merge(final,voter,on=['year','state'])
final.head()


fips = house_filtered.loc[:,['state','state_fips','year']]
final = pd.merge(fips,final,on=['state','year'])
final.head()


################### splitting data ###################
x_train = final.loc[final['year'] < 2010,final.columns!='voter_turnout']
y_train = final.loc[final['year'] < 2010,final.columns =='voter_turnout']

x_test = final.loc[final['year'] >= 2010,final.columns!='voter_turnout']
y_test = final.loc[final['year'] >= 2010, ['state','year','voter_turnout']] ## state and year for visualizing 

x_train.drop('state',axis=1,inplace=True)
x_test.drop('state',axis=1,inplace=True)

x_train.drop('year',axis=1,inplace=True)
x_test.drop('year',axis=1,inplace=True)

print(x_train.shape,y_train.shape,x_test.shape,y_test.shape)

(200, 9) (200, 1) (100, 9) (100, 3)


%%time
gb = GradientBoostingRegressor()

params_gb_reg = {
    "n_estimators":[10,50,100,200],
    "learning_rate":[0.1,0.3,0.5,0.7,1],
    "criterion" : ['mse']
}


cv = GridSearchCV(gb,params_gb_reg,cv=10,n_jobs=-1)
cv.fit(x_train,y_train)

Wall time: 6.33 s

GridSearchCV(cv=10, estimator=GradientBoostingRegressor(), n_jobs=-1,
             param_grid={'criterion': ['mse'],
                         'learning_rate': [0.1, 0.3, 0.5, 0.7, 1],
                         'n_estimators': [10, 50, 100, 200]})


cv.best_params_

{'criterion': 'mse', 'learning_rate': 0.1, 'n_estimators': 10}


best_gb = cv.best_estimator_
y_gb = best_gb.predict(x_test)
gb_mse = round(mean_squared_error(y_test['voter_turnout'],y_gb),5)
print("Gradient Boost MSE",gb_mse)

Gradient Boost MSE 0.00811


ft_names_c = final.columns.to_list()
ft_names_c.remove('state')
ft_names_c.remove('voter_turnout')
ft_names_c.remove('year')
ft_names_c
feature_imp_gb_reg = pd.Series(best_gb.feature_importances_,index=ft_names_c).sort_values(ascending=False)
feature_imp_gb_reg = feature_imp_gb_reg.head(len(feature_imp_gb_reg))


# Creating a bar plot for feature importance
plt.figure(figsize=(9,9))
sns.barplot(x=feature_imp_gb_reg, y=feature_imp_gb_reg.index,palette='dark:salmon')
plt.grid(axis='x', alpha=0.5, color='lightgrey')
# Add labels to your graph
plt.xlabel('Feature Importance Score',fontsize=12,color='maroon')
plt.ylabel('Features',fontsize=12,color='maroon')
plt.title("Visualizing Important Features - GradientBoost after tuning", fontsize=15,color='maroon')
plt.show()


plt.figure(figsize=(9,9))
plt.figure(figsize=(9,9))
plt.scatter(y_test.voter_turnout,y_gb,c='salmon')
plt.xlabel('True Values', fontsize=15)
plt.ylabel('Predictions', fontsize=15)
p1 = max(y_gb.max(), y_test.voter_turnout.max())
p2 = min(y_gb.min(), y_test.voter_turnout.min())
plt.plot([p1, p2], [p1, p2], 'b-')
plt.title("Gradient Boost Predictions vs True Values",fontsize=15)
plt.show()

<Figure size 648x648 with 0 Axes>


plt.figure(figsize=(10,7))
x_ax=range(200)
plt.scatter(range(len(y_test.voter_turnout)), y_test.voter_turnout, s=5, color="#00239CFF", label="original")
plt.plot(range(len(y_test.voter_turnout)), y_gb, lw=1.5, color="#E10600FF", label="predicted")
plt.legend()
plt.xlabel('Samples')
plt.ylabel('Voter turnout')
plt.title('Predicted vs True turnout percentages - Gradient Boost')
plt.show()


%%time
rf = RandomForestRegressor()

# Create the random grid
rf_grid = {'n_estimators':[int(x) for x in np.linspace(start = 100, stop = 500, num = 10)],
               'max_depth': [2,3,4,5,7,10,20],
               'min_samples_split': [2, 5, 10],
               'bootstrap': [True, False]}

rf_cv = GridSearchCV(rf,rf_grid,cv=5,n_jobs=-1)
rf_cv.fit(x_train,y_train)

Wall time: 4min 13s

GridSearchCV(cv=5, estimator=RandomForestRegressor(), n_jobs=-1,
             param_grid={'bootstrap': [True, False],
                         'max_depth': [2, 3, 4, 5, 7, 10, 20],
                         'min_samples_split': [2, 5, 10],
                         'n_estimators': [100, 144, 188, 233, 277, 322, 366,
                                          411, 455, 500]})


rf_cv.best_params_

{'bootstrap': True,
 'max_depth': 3,
 'min_samples_split': 10,
 'n_estimators': 144}


best_rf = rf_cv.best_estimator_
y_rf = best_rf.predict(x_test)
rf_mse = round(mean_squared_error(y_test['voter_turnout'],y_rf),5)
print("Random Forrest Regressor MSE",rf_mse)

Random Forrest Regressor MSE 0.00761


feature_imp_rf = pd.Series(best_rf.feature_importances_,index=ft_names_c).sort_values(ascending=False)
feature_imp_rf = feature_imp_rf.head(len(feature_imp_rf))


# Creating a bar plot for feature importance
plt.figure(figsize=(9,9))
sns.barplot(x=feature_imp_rf, y=feature_imp_rf.index,palette='dark:salmon')
plt.grid(axis='x', alpha=0.5, color='lightgrey')
# Add labels to your graph
plt.xlabel('Feature Importance Score',fontsize=12,color='maroon')
plt.ylabel('Features',fontsize=12,color='maroon')
plt.title("Visualizing Important Features - Random Forrest", fontsize=15,color='maroon')
plt.show()


plt.figure(figsize=(9,9))
plt.figure(figsize=(9,9))
plt.scatter(y_test.voter_turnout,y_rf,c='salmon')
plt.xlabel('True Values', fontsize=15)
plt.ylabel('Predictions', fontsize=15)
p1 = max(y_rf.max(), y_test.voter_turnout.max())
p2 = min(y_rf.min(), y_test.voter_turnout.min())
plt.plot([p1, p2], [p1, p2], 'b-')
plt.title("Random Forrest Regressor Predictions vs True Values",fontsize=15)
plt.show()

<Figure size 648x648 with 0 Axes>


plt.figure(figsize=(10,7))
x_ax=range(200)
plt.scatter(range(len(y_test.voter_turnout)), y_test.voter_turnout.values, s=5, color="#00239CFF", label="original")
plt.plot(range(len(y_test.voter_turnout)), y_rf, lw=1.5, color="#E10600FF", label="predicted")
plt.legend()
plt.xlabel('Samples')
plt.ylabel('Voter turnout')
plt.title('Predicted vs True turnout percentages - Random Forrest')
plt.show()


%%time
################### scaling data ###################
scaler = StandardScaler()
x_train1 = scaler.fit_transform(x_train)
x_test1 = scaler.fit_transform(x_test)

knn_reg = KNeighborsRegressor()

knn_params = {
    'n_neighbors': [2,3,4,5,6,7,8,9,10,11,12],
    'weights': ['uniform', 'distance'],  
    'p': [1,2],
    'algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute']
    }

knn_cv = GridSearchCV(estimator=knn_reg, param_grid=knn_params, cv=5, n_jobs=-1)
knn_cv.fit(x_train1, y_train)

Wall time: 1.29 s

GridSearchCV(cv=5, estimator=KNeighborsRegressor(), n_jobs=-1,
             param_grid={'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
                         'n_neighbors': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12],
                         'p': [1, 2], 'weights': ['uniform', 'distance']})


knn_cv.best_params_

{'algorithm': 'auto', 'n_neighbors': 3, 'p': 1, 'weights': 'uniform'}


best_knn = knn_cv.best_estimator_
best_knn.fit(x_train1,y_train)
y_knn = best_knn.predict(x_test1)
knn_mse = round(mean_squared_error(y_test['voter_turnout'],y_knn),5)
print("KNN Regressor MSE",knn_mse)

KNN Regressor MSE 0.00555


plt.figure(figsize=(9,9))
plt.figure(figsize=(9,9))
plt.scatter(y_test.voter_turnout,y_knn,c='salmon')
plt.xlabel('True Values', fontsize=15)
plt.ylabel('Predictions', fontsize=15)
p1 = max(y_knn.max(), y_test.voter_turnout.max())
p2 = min(y_knn.min(), y_test.voter_turnout.min())
plt.plot([p1, p2], [p1, p2], 'b-')
plt.title("KNN Regressor Predictions vs True Values",fontsize=15)
plt.show()

<Figure size 648x648 with 0 Axes>


plt.figure(figsize=(10,7))
x_ax=range(200)
plt.scatter(range(len(y_test.voter_turnout.values)), y_test.voter_turnout.values, s=5, color="#00239CFF", label="original")
plt.plot(range(len(y_test.voter_turnout.values)), y_knn, lw=1.5, color="#E10600FF", label="predicted")
plt.legend()
plt.xlabel('Samples')
plt.ylabel('Voter turnout')
plt.title('Predicted vs True turnout percentages - KNN')
plt.show()


log_regs=["Regressor", "MSE"]
log_reg = pd.DataFrame(columns=log_regs)
log_entry = pd.DataFrame([[gb.__class__.__name__, gb_mse]], columns=log_regs)
log_reg = log_reg.append(log_entry)

log_entry = pd.DataFrame([[rf.__class__.__name__, rf_mse]], columns=log_regs)
log_reg = log_reg.append(log_entry)

log_entry = pd.DataFrame([[knn_reg.__class__.__name__, knn_mse]], columns=log_regs)
log_reg = log_reg.append(log_entry)
log_reg


sns.set_color_codes("muted")
sns.barplot(x='MSE', y='Regressor', data=log_reg, color="m")

plt.xlabel('MSE')
plt.title('Regressor Accuracy')
plt.show()


viz_states = ['California','Florida','South Dakota','Wyoming']
states_fips = [6, 12, 46, 56]


y_pred = y_knn.ravel()
viz_data = y_test.assign(predicted_voter_turnout = y_pred)
viz_data = viz_data.loc[viz_data['state'].isin(viz_states)]
viz_data.reset_index(inplace=True,drop=True)
viz_data


fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('True Values', fontsize = 15)
ax.set_ylabel('Predections', fontsize = 15)
ax.set_title('True Values vs Predections - 2010', fontsize = 20)
colors = ['#051181', '#fb3c3c', '#feb64a','#128b8e']
y_t = []
y_p = []
for state, color in zip(viz_states,colors):
    data_y1 =  viz_data.loc[viz_data['year']==2010, :]
    pred = data_y1.loc[data_y1['state'] == state, data_y1.columns == 'predicted_voter_turnout']
    true = data_y1.loc[data_y1['state'] == state, data_y1.columns == 'voter_turnout']
    y_t.append(true.values)
    y_p.append(pred.values)
    ax.scatter(true, pred, c = color, s = 50)
    
ax.legend(viz_states)
ax.grid()
p1 = max(max(y_t), max(y_p)).ravel()
p2 = min(min(y_t), min(y_p)).ravel()
plt.plot([p1, p2], [p1, p2], 'b-')
plt.show()



fig = plt.figure(figsize = (8,8))
ax = fig.add_subplot(1,1,1) 
ax.set_xlabel('True Values', fontsize = 15)
ax.set_ylabel('Predections', fontsize = 15)
ax.set_title('True Values vs Predections - 2012', fontsize = 20)
y_t = []
y_p = []
for state, color in zip(viz_states,colors):
    data_y1 =  viz_data.loc[viz_data['year']==2012, :]
    pred = data_y1.loc[data_y1['state'] == state, data_y1.columns == 'predicted_voter_turnout']
    true = data_y1.loc[data_y1['state'] == state, data_y1.columns == 'voter_turnout']
    y_t.append(true.values)
    y_p.append(pred.values)
    ax.scatter(true, pred, c = color, s = 50)
    
ax.legend(viz_states)
ax.grid()
p1 = max(max(y_t), max(y_p)).ravel()
p2 = min(min(y_t), min(y_p)).ravel()
plt.plot([p1, p2], [p1, p2], 'b-')
plt.show()


y_pred = y_rf.ravel()
viz_data = y_test.assign(predicted_voter_turnout = y_pred)
viz_data.reset_index(inplace=True,drop=True)
viz_data['mse'] = viz_data.apply(lambda row : pow((row['voter_turnout'] - row['predicted_voter_turnout']),2),axis=1)
viz_data['total_mse'] = viz_data.groupby(['state'])['mse'].transform(sum)

viz_data['mean_true_turnout'] = viz_data.groupby(['state'])['voter_turnout'].transform(sum)
viz_data['mean_pred_turnout'] = viz_data.groupby(['state'])['predicted_voter_turnout'].transform(sum)
viz_data['mean_true_turnout'] = viz_data.apply(lambda row : round((row['mean_true_turnout']/2),3),axis=1)
viz_data['mean_pred_turnout'] = viz_data.apply(lambda row : round((row['mean_pred_turnout']/2),3),axis=1)

viz_data.drop_duplicates('state',inplace=True)
viz_data.drop(['voter_turnout','mse','predicted_voter_turnout','year'],inplace=True,axis=1)
viz_data.sort_values('total_mse',ascending=True,inplace=True)
viz_data.reset_index(inplace=True,drop=True)
least_five = viz_data.tail(5)
least_five


y_pred = y_gb.ravel()
viz_data = y_test.assign(predicted_voter_turnout = y_pred)
viz_data.reset_index(inplace=True,drop=True)
# viz_data = viz_data.loc[viz_data.year==2012,:]
viz_data['mse'] = viz_data.apply(lambda row : pow((row['voter_turnout'] - row['predicted_voter_turnout']),2),axis=1)
viz_data['total_mse'] = viz_data.groupby(['state'])['mse'].transform(sum)

viz_data['mean_true_turnout'] = viz_data.groupby(['state'])['voter_turnout'].transform(sum)
viz_data['mean_pred_turnout'] = viz_data.groupby(['state'])['predicted_voter_turnout'].transform(sum)
viz_data['mean_true_turnout'] = viz_data.apply(lambda row : round((row['mean_true_turnout']/2),3),axis=1)
viz_data['mean_pred_turnout'] = viz_data.apply(lambda row : round((row['mean_pred_turnout']/2),3),axis=1)

viz_data.drop_duplicates('state',inplace=True)
viz_data.drop(['voter_turnout','mse','predicted_voter_turnout','year'],inplace=True,axis=1)
viz_data.sort_values('total_mse',ascending=True,inplace=True)
viz_data.reset_index(inplace=True,drop=True)
least_five = viz_data.tail(5)
least_five


viz_data.head(25)


final.columns

Index(['state', 'state_fips', 'year', 'bachelors', 'unemployed', 'poverty',
       'hs_grad', 'per_capita_income', 'white', 'black', 'hispanic',
       'voter_turnout'],
      dtype='object')


################### splitting data ###################

ft_drop = ['voter_turnout','per_capita_income','bachelors','hs_grad']
ft_stay = set(final.columns) - set(ft_drop)
ft_stay

x_train = final.loc[final['year'] < 2010, ft_stay]
y_train = final.loc[final['year'] < 2010,final.columns =='voter_turnout']


x_test = final.loc[final['year'] >= 2010,ft_stay]
y_test = final.loc[final['year'] >= 2010, ['state','year','voter_turnout']] ## state and year for visualizing 

x_train.drop('state',axis=1,inplace=True)
x_test.drop('state',axis=1,inplace=True)

# x_train.drop('year',axis=1,inplace=True)
# x_test.drop('year',axis=1,inplace=True)

print(x_train.shape,y_train.shape,x_test.shape,y_test.shape)

(200, 7) (200, 1) (100, 7) (100, 3)


%%time

gb = GradientBoostingRegressor()

params_gb_reg = {
    "n_estimators":[10,50,100,200],
    "learning_rate":[0.1,0.3,0.5,0.7,1],
    "criterion" : ['mse']
}


cv = GridSearchCV(gb,params_gb_reg,cv=10,n_jobs=-1)
cv.fit(x_train,y_train)

print(cv.best_params_)

best_gb = cv.best_estimator_
y_gb1 = best_gb.predict(x_test)
gb_mse = round(mean_squared_error(y_test['voter_turnout'],y_gb1),5)
print("Gradient Boost MSE",gb_mse)


feature_imp_gb_reg = pd.Series(best_gb.feature_importances_,index=x_train.columns).sort_values(ascending=False)
feature_imp_gb_reg = feature_imp_gb_reg.head(len(feature_imp_gb_reg))


# Creating a bar plot for feature importance
plt.figure(figsize=(9,9))
sns.barplot(x=feature_imp_gb_reg, y=feature_imp_gb_reg.index,palette='dark:salmon')
plt.grid(axis='x', alpha=0.5, color='lightgrey')
# Add labels to your graph
plt.xlabel('Feature Importance Score',fontsize=12,color='maroon')
plt.ylabel('Features',fontsize=12,color='maroon')
plt.title("Visualizing Important Features - GradientBoost after tuning", fontsize=15,color='maroon')
plt.show()

{'criterion': 'mse', 'learning_rate': 0.3, 'n_estimators': 100}
Gradient Boost MSE 0.01461

Wall time: 3.36 s


y_pred = y_gb1.ravel()
viz_data = y_test.assign(predicted_voter_turnout = y_pred)
# viz_data = viz_data.loc[viz_data.year==2012,:]
viz_data.reset_index(inplace=True,drop=True)
viz_data['mse'] = viz_data.apply(lambda row : pow((row['voter_turnout'] - row['predicted_voter_turnout']),2),axis=1)
viz_data['total_mse'] = viz_data.groupby(['state'])['mse'].transform(sum)
viz_data['mean_true_turnout'] = viz_data.groupby(['state'])['voter_turnout'].transform(sum)
viz_data['mean_pred_turnout'] = viz_data.groupby(['state'])['predicted_voter_turnout'].transform(sum)
viz_data['mean_true_turnout'] = viz_data.apply(lambda row : round((row['mean_true_turnout']/2),3),axis=1)
viz_data['mean_pred_turnout'] = viz_data.apply(lambda row : round((row['mean_pred_turnout']/2),3),axis=1)
viz_data.drop_duplicates('state',inplace=True)
viz_data.drop(['voter_turnout','mse','predicted_voter_turnout','year'],inplace=True,axis=1)
viz_data.sort_values('total_mse',ascending=True,inplace=True)
viz_data.reset_index(inplace=True,drop=True)
top_25 = viz_data.head(25)

for i in top_25['state']:
    if i in list(least_five['state']):
            print(i,"moved to top 25") 
            
viz_data.head(25)

Texas moved to top 25
Minnesota moved to top 25


least_five


## pre-processing


house_data = pd.read_csv("USElectionResults19762020/1976-2020-house.csv")


house_data.head()
house_data.shape

(31103, 20)


## candidates who's values is 'writein' will be dropped because they are not a part of Democratic or Republican parties
house_data.replace(" ",None)
drop = house_data[house_data['candidate'] == 'WRITEIN'].index
house_data.drop(drop,inplace=True)
house_data.shape

(28546, 20)


house_data.isnull().sum()

year                 0
state                0
state_po             0
state_fips           0
state_cen            0
state_ic             0
office               0
district             0
stage                0
runoff            7845
special              0
candidate            0
party             1065
writein              0
mode                 0
candidatevotes       0
totalvotes           0
unofficial           0
version              0
fusion_ticket        0
dtype: int64


house_data.runoff.value_counts().plot(kind='barh')
plt.title('True vs False values in Runoff feature')
plt.xlabel('Count')
plt.show()

house_data.drop('runoff',inplace=True,axis=1)
house_data.head()


house_data = house_data.dropna()


house_data.shape

(27481, 19)


house_data['state'] = house_data['state'].str.title()


house_data.head()


house_data = drop_DC(house_data)
house_data.shape

(27471, 19)


house_data.reset_index(inplace=True,drop=True)
house_data.head()


## preparing data


## adding sum_per_party columns, which sums the votes that each party got in each state 
house_data['sum_per_party'] = house_data.groupby(['state','party','year'])['candidatevotes'].transform(sum)
house_data


## for each state, the highest sum of votes will be kept 
house_grouped = house_data.loc[house_data.groupby(['state','year'])['sum_per_party'].idxmax()]
house_grouped.reset_index(inplace=True,drop=True)


house_grouped


## creating label column 
for i,r in house_grouped.iterrows():
    year = r.year
    state = r.state
    label = r.party
    house_grouped.loc[i, 'label'] = label


house_grouped.label.unique()

array(['DEMOCRAT', 'REPUBLICAN', 'DEMOCRATIC-FARMER-LABOR', 'INDEPENDENT'],
      dtype=object)


to_drop = ['DEMOCRATIC-FARMER-LABOR','INDEPENDENT']
house_grouped = house_grouped.loc[~house_grouped['label'].isin(to_drop)]
house_grouped


house_grouped.replace(['DEMOCRAT','REPUBLICAN'],[1,0],inplace=True)
house_grouped.reset_index(inplace=True,drop=True)
house_grouped


## we will drop features that has only one value
to_drop = []
for i in house_grouped.columns:
    if len(house_grouped[i].unique()) < 2:
        to_drop.append(i)
to_drop

['office', 'stage', 'special', 'mode', 'version']


house_grouped.drop(to_drop,inplace=True,axis=1)
house_grouped.shape

(1133, 16)


## dropping features that built the label and other irrelevant features
house_grouped.drop(['candidatevotes','state_po','candidate','sum_per_party','party','state_cen','state_ic','district'],inplace=True,axis=1)
house_grouped


fig = plt.figure(figsize=(10, 8))
fig.subplots_adjust(hspace=0.4, wspace=0.3)
colors = plt.get_cmap('tab10').colors

for idx, ft, color in zip(range(3),['writein','unofficial','fusion_ticket'], colors):
    ax = fig.add_subplot(2, 2, idx+1)
    house_grouped[ft].value_counts().plot(kind='barh',color = color)
    plt.title('True vs False values in '+ ft+' feature')
    plt.xlabel('Count')
    plt.show()


house_grouped.drop(['writein','unofficial','fusion_ticket'],inplace=True,axis=1)
house_grouped.shape

(1133, 5)


democrats = []
republicans = []
years_group = house_grouped.year.unique()

for i in years_group:
    democrats.append(house_grouped[house_grouped['year'] == i]['label'].value_counts()[1])
    republicans.append(house_grouped[house_grouped['year'] == i]['label'].value_counts()[0])

plt.figure(figsize=(20, 10))
X_axis = np.arange(len(years_group))
plt.bar(X_axis - 0.2, democrats, 0.3, label = 'Democrats',color='#FC766AFF')
plt.bar(X_axis + 0.2, republicans, 0.3, label = 'Republicans')
  
plt.xticks(X_axis, years_group)
plt.xlabel("Years")
plt.ylabel("Number of states")
plt.title("Number of States Who Was Marked as Democrats/Republicans in each year")
plt.legend()
plt.show()


so_ec_fts = ['bachelors_2019','bachelors_2010','unemployed_2010','unemployed_2017',
'poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019','per_capita_income_2010',
'per_capita_income_2019']
pops = ['pop2010','pop2017','pop_2019']
democ = county_complete.loc[:,['state','name']+pops+['white_2010','white_2019','black_2010','black_2019','hispanic_2010'
                                ,'hispanic_2019']+so_ec_fts]
democ.isnull().sum()

state                      0
name                       0
pop2010                    0
pop2017                    3
pop_2019                   0
white_2010                 0
white_2019                 0
black_2010                17
black_2019                 0
hispanic_2010              0
hispanic_2019              0
bachelors_2019             0
bachelors_2010             0
unemployed_2010            3
unemployed_2017            3
poverty_2017               2
poverty_2010               0
hs_grad_2010               0
hs_grad_2019               0
per_capita_income_2010     0
per_capita_income_2019     0
dtype: int64


imputer = KNNImputer(n_neighbors=5, weights="distance")
imputed = imputer.fit_transform(democ.drop(['state','name'],axis=1))

# replace values in the dataset with new imputed values
democ[democ.columns.drop(['name','state'])] = imputed


democ.rename(columns={'pop_2019':'pop2019'},inplace=True)
democ.head()


##converting these features from percentages to numeric population numbers

perc_toNumeric = ['bachelors_2019','bachelors_2010','poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019',
               'white_2010','white_2019','black_2010','black_2019','hispanic_2010','hispanic_2019']

for i in perc_toNumeric:
    index = i.split('_')
    democ[i] = democ.apply(lambda row : int((row[i]/100)*row['pop'+index[-1]]),axis=1)

democ.head()


## group by state and summing the features
democ_grouped = democ.groupby(['state']).sum()
democ_grouped.reset_index(inplace=True)
democ_grouped.head()


## finding features data in missing years

all_fts = ['white','black','hispanic','bachelors', 'unemployed', 'poverty', 'hs_grad', 'per_capita_income']
for i in range(2000,2022,2): ## change 2022 -> 2024
    for j in all_fts:
        ft = j+'_'
        if(ft +str(i) not in democ_grouped.columns):
            if ft+'2010' in cols:
                pop1 = ft+'2010'
                y1 = 2010
                if ft+'2017' in cols:
                    pop2 = ft+'2017'
                    y2=2017
                else:
                    pop2 = ft+'2019'
                    y2 = 2019
            democ_grouped[ft+str(i)] = democ_grouped.apply(lambda row : (estimated_pop_based_on_gr(i,row[pop1],row[pop2],y1,y2)),axis=1)

            
            
democ_grouped.head()


## dropping odd years because they are not included in elections
democ_grouped.drop(['pop2010','pop2017','pop2019','white_2019','black_2019','hispanic_2019',
                   'bachelors_2019','unemployed_2017','poverty_2017','hs_grad_2019','per_capita_income_2019',
                   ],inplace=True,axis=1)


all_fts

['white',
 'black',
 'hispanic',
 'bachelors',
 'unemployed',
 'poverty',
 'hs_grad',
 'per_capita_income']


democ_grouped.columns

Index(['state', 'white_2010', 'black_2010', 'hispanic_2010', 'bachelors_2010',
       'unemployed_2010', 'poverty_2010', 'hs_grad_2010',
       'per_capita_income_2010', 'white_2000', 'black_2000', 'hispanic_2000',
       'bachelors_2000', 'unemployed_2000', 'poverty_2000', 'hs_grad_2000',
       'per_capita_income_2000', 'white_2002', 'black_2002', 'hispanic_2002',
       'bachelors_2002', 'unemployed_2002', 'poverty_2002', 'hs_grad_2002',
       'per_capita_income_2002', 'white_2004', 'black_2004', 'hispanic_2004',
       'bachelors_2004', 'unemployed_2004', 'poverty_2004', 'hs_grad_2004',
       'per_capita_income_2004', 'white_2006', 'black_2006', 'hispanic_2006',
       'bachelors_2006', 'unemployed_2006', 'poverty_2006', 'hs_grad_2006',
       'per_capita_income_2006', 'white_2008', 'black_2008', 'hispanic_2008',
       'bachelors_2008', 'unemployed_2008', 'poverty_2008', 'hs_grad_2008',
       'per_capita_income_2008', 'white_2012', 'black_2012', 'hispanic_2012',
       'bachelors_2012', 'unemployed_2012', 'poverty_2012', 'hs_grad_2012',
       'per_capita_income_2012', 'white_2014', 'black_2014', 'hispanic_2014',
       'bachelors_2014', 'unemployed_2014', 'poverty_2014', 'hs_grad_2014',
       'per_capita_income_2014', 'white_2016', 'black_2016', 'hispanic_2016',
       'bachelors_2016', 'unemployed_2016', 'poverty_2016', 'hs_grad_2016',
       'per_capita_income_2016', 'white_2018', 'black_2018', 'hispanic_2018',
       'bachelors_2018', 'unemployed_2018', 'poverty_2018', 'hs_grad_2018',
       'per_capita_income_2018', 'white_2020', 'black_2020', 'hispanic_2020',
       'bachelors_2020', 'unemployed_2020', 'poverty_2020', 'hs_grad_2020',
       'per_capita_income_2020'],
      dtype='object')


final_d = pd.DataFrame(columns=['year','state']+all_fts)
row_data = 0
index=0
for i in range(2000,2022,2):
    row_data=0
    new_row = []
    new_row.append(i)
    for j in democ_grouped.state.unique():
        new_row.append(j)
        for ft in all_fts:
            val= democ_grouped.loc[row_data,[ft+"_"+str(i)]]
            new_row.append(val[0])            
        final_d.loc[index] = new_row
        new_row = []
        new_row.append(i)
        index+=1
        row_data+=1
final_d


final_d = drop_DC(final_d)
final_d


house_grouped = house_grouped[house_grouped.year>=2000]
house_grouped


final_d = pd.merge(final_d,house_grouped,on=['state','year'])


final_d


def get_metrics(y_test, y_pred):
    
    actual_pos = y_test == 1
    actual_neg = y_test == 0
    
    # get confusion matrix
    mat = metrics.confusion_matrix(y_test, y_pred)
    true_neg, false_pos, false_neg, true_pos = mat.ravel()
    
    # calculate sensitivity and specificity
    sensitivity = round(true_pos / np.sum(actual_pos), 3)
    specificity = round(true_neg / np.sum(actual_neg), 3)
    
    return sensitivity, specificity


# splitting data
ft_names_d = final_d.columns.drop(['label','state']).to_list()
x = final_d.drop(['state','label'],axis=1)
y = final_d['label']
xc_train, xc_test, yc_train, yc_test = train_test_split(x, y, test_size=0.25, random_state=None)
print(xc_train.shape,yc_train.shape,xc_test.shape,yc_test.shape)

(407, 11) (407,) (136, 11) (136,)


%%time
##scaling for svc

scaler = StandardScaler()
xc_train1 = scaler.fit_transform(xc_train) 
xc_test1 = scaler.fit_transform(xc_test) 

svc=SVC()
svc_params = [{'kernel':['linear'],'C': [0.001, 0.01,0.05,0.1]},
             {'gamma': [0.01,0.02,0.03,0.04,0.05,1,5,9]}]
model_svc = GridSearchCV(svc, svc_params,cv=5,scoring='accuracy',verbose=3,n_jobs=-1)
model_svc.fit(xc_train1, yc_train)

print(model_svc.best_params_)

Fitting 5 folds for each of 12 candidates, totalling 60 fits
{'gamma': 1}
Wall time: 236 ms


y_pred_svc=model_svc.predict(xc_test1)
acc_svc = accuracy_score(yc_test,y_pred_svc)
print('Accuracy Score:')
print(acc_svc)

Accuracy Score:
0.7647058823529411


sensitivity, specificity = get_metrics(yc_test, y_pred_svc)

svc_df = pd.DataFrame([acc_svc, sensitivity, specificity]).T
svc_df = svc_df.rename(index={0: 'SVC'}, columns={0: 'Accuracy', 1: 'Sensitivity', 2: 'Specificity'})
svc_df


plot_confusion_matrix(model_svc.best_estimator_, xc_test1, yc_test, cmap='Blues')
plt.title("SVC Confusion Matrix")
plt.show()


%%time
ab_clf = AdaBoostClassifier()

ab_params = {
    'n_estimators': [50, 100, 200, 250, 300],
    'learning_rate'  : [0.1, 0.3, 0.5, 0.7, 1.0]
}

ab_cv = GridSearchCV(ab_clf, ab_params, cv=5, verbose=1, n_jobs=-1)
ab_cv.fit(xc_train, yc_train)

print("Best: %f using %s" % (ab_cv.best_score_, ab_cv.best_params_))

Fitting 5 folds for each of 25 candidates, totalling 125 fits
Best: 0.818308 using {'learning_rate': 1.0, 'n_estimators': 300}
Wall time: 15.9 s


y_pred_ab = ab_cv.predict(xc_test)
acc_ab = round(accuracy_score(yc_test, y_pred_ab), 3)
print("AdaBoost Classifier Model Accuracy:", acc_ab)

# calculate metrics

sensitivity, specificity = get_metrics(yc_test, y_pred_ab)

# display metrics
ab_df = pd.DataFrame([acc_ab, sensitivity, specificity]).T
ab_df = ab_df.rename(index={0: 'AdaBoost Classifier'}, columns={0: 'Accuracy', 1: 'Sensitivity', 2: 'Specificity'})
ab_df

AdaBoost Classifier Model Accuracy: 0.853


best_ab = ab_cv.best_estimator_

plot_confusion_matrix(best_ab, xc_test, yc_test, cmap='BuPu')
plt.title("AdaBoost Confusion Matrix")
plt.show()


feature_imp_ab = pd.Series(best_ab.feature_importances_,index=xc_train.columns).sort_values(ascending=False)
feature_imp_ab = feature_imp_ab.head(int(len(feature_imp_ab)))

# Creating a bar plot for feature importance
sns.barplot(x=feature_imp_ab, y=feature_imp_ab.index,palette='Blues_d')
plt.grid(axis='x', alpha=0.5, color='lightgrey')
# Add labels to your graph
plt.xlabel('Feature Importance Score',fontsize=12,color='maroon')
plt.ylabel('Features',fontsize=12,color='maroon')
plt.title("Visualizing Important Features - AdaBoost", fontsize=15,color='maroon')
plt.show()


%%time
rf_1=RandomForestClassifier()

# Create the param grid
rf_params = {'n_estimators': [10,50,100,200,300],
               'criterion': ['gini'], 
               'max_depth': [4,5,6,7,8,9,10],
               'min_samples_leaf': [1,3,5]}



rf_cv = GridSearchCV(
    rf_1, rf_params,
    cv=5, n_jobs=-1,
)
rf_cv.fit(xc_train, yc_train)

print('Best accuracy: %.3f' % rf_cv.best_score_)
print('\nBest params:\n', rf_cv.best_params_)

Best accuracy: 0.828

Best params:
 {'criterion': 'gini', 'max_depth': 10, 'min_samples_leaf': 1, 'n_estimators': 200}
Wall time: 38.4 s


y_pred_rf = rf_cv.predict(xc_test)
acc_rf = round(accuracy_score(yc_test, y_pred_rf), 5)
print("Random Forest Classifier Model Accuracy (after tuning parameters) :", acc_rf)

# display metrics
rf_df = pd.DataFrame([acc_rf, sensitivity, specificity]).T
rf_df = rf_df.rename(index={0: 'Random Forrest Classifier'}, columns={0: 'Accuracy', 1: 'Sensitivity', 2: 'Specificity'})
rf_df

Random Forest Classifier Model Accuracy (after tuning parameters) : 0.875


plot_confusion_matrix(rf_cv.best_estimator_, xc_test, yc_test, cmap='Blues')
plt.title("Random Forest Confusion Matrix")
plt.show()


feature_imp_rf = pd.Series(rf_cv.best_estimator_.feature_importances_,index=ft_names_d).sort_values(ascending=False)
feature_imp_rf = feature_imp_rf.head(int(len(feature_imp_rf)))

# Creating a bar plot for feature importance
sns.barplot(x=feature_imp_rf, y=feature_imp_rf.index,palette='Blues_d')
plt.grid(axis='x', alpha=0.5, color='lightgrey')
# Add labels to your graph
plt.xlabel('Feature Importance Score',fontsize=12,color='maroon')
plt.ylabel('Features',fontsize=12,color='maroon')
plt.title("Visualizing Important Features - Random Forrest Classifier", fontsize=15,color='maroon')
plt.show()


%%time
knn_clf = KNeighborsClassifier()

knn_clf_params = { 'n_neighbors' : [1,3,5,7,9,11,13,15],
               'weights' : ['uniform','distance'],
               'metric' : ['minkowski','euclidean','manhattan']}

knn_clf_cv = GridSearchCV(knn_clf, knn_clf_params, verbose = 1, cv=5, n_jobs = -1)

knn_clf_res = knn_clf_cv.fit(xc_train1, yc_train)
knn_clf_res.best_params_

Fitting 5 folds for each of 48 candidates, totalling 240 fits
Wall time: 314 ms

{'metric': 'manhattan', 'n_neighbors': 9, 'weights': 'distance'}


y_pred_knn_clf = knn_clf_res.best_estimator_.predict(xc_test1)
acc_knn = round(accuracy_score(yc_test, y_pred_knn_clf), 5)
print("KNN Classifier Model Accuracy :", acc_knn)

# display metrics
knn_df = pd.DataFrame([acc_knn, sensitivity, specificity]).T
knn_df = knn_df.rename(index={0: 'KNN Classifier'}, columns={0: 'Accuracy', 1: 'Sensitivity', 2: 'Specificity'})
knn_df

KNN Classifier Model Accuracy : 0.78676


plot_confusion_matrix(knn_clf_cv.best_estimator_, xc_test1, yc_test, cmap='Blues')
plt.title("Random Forest Confusion Matrix")
plt.show()


log_cols=["Classifier", "Accuracy"]
log = pd.DataFrame(columns=log_cols)
log_entry = pd.DataFrame([[svc.__class__.__name__, acc_svc*100]], columns=log_cols)
log = log.append(log_entry)

log_entry = pd.DataFrame([[ab_clf.__class__.__name__, acc_ab*100]], columns=log_cols)
log = log.append(log_entry)

log_entry = pd.DataFrame([[rf_1.__class__.__name__, acc_rf*100]], columns=log_cols)
log = log.append(log_entry)

log_entry = pd.DataFrame([[knn_clf.__class__.__name__, acc_knn*100]], columns=log_cols)
log = log.append(log_entry)
log


sns.set_color_codes("muted")
sns.barplot(x='Accuracy', y='Classifier', data=log, color="salmon")

plt.xlabel('Accuracy %')
plt.title('Comparison Between Models')
plt.show()


senate_data = pd.read_csv("USElectionResults19762020/1976-2020-senate.csv")
senate_data.head()
senate_data.shape

(3629, 19)


##dropping undefined candidates 
senate_data.replace(" ",None)
drop =( senate_data[house_data['candidate'] == 'OTHER'].index | senate_data[senate_data['candidate'] == 'NA'].index | 
       senate_data[senate_data['candidate'] == 'SCATTER'].index)
senate_data.drop(drop,inplace=True)
senate_data.shape

(3603, 19)


senate_data.isnull().sum()

year                  0
state                 0
state_po              0
state_fips            0
state_cen             0
state_ic              0
office                0
district              0
stage                 0
special               0
candidate           423
party_detailed      573
writein               0
mode                  0
candidatevotes        0
totalvotes            0
unofficial            0
version               0
party_simplified      0
dtype: int64


senate_data = senate_data.dropna()
senate_data.shape

(3030, 19)


senate_data['state'] = senate_data['state'].str.title()
senate_data = drop_DC(senate_data)
senate_data.reset_index(inplace=True,drop=True)
senate_data.shape

(3030, 19)


# Since we are told Independents count as Democrats, we will convert their Values to Democrats
senate_data.replace({'INDEPENDENT' : 'DEMOCRAT'},inplace=True)


senate_data.party_detailed.unique()

array(['REPUBLICAN', 'DEMOCRAT', 'LIBERTARIAN', 'AMERICAN INDEPENDENT',
       'PEACE AND FREEDOM', 'AMERICAN', 'PROHIBITION', 'NONE',
       'U.S. LABOR', 'SOCIALIST WORKERS', 'HUMAN RIGHTS',
       'SOCIALIST LABOR', 'COMMUNIST', 'INDEPENDENT AMERICAN', 'LABOR',
       'LA RAZA UNIDA', 'LIBERAL', 'CONSERVATIVE', 'CONSTITUTION',
       'LIBERTY UNION', 'DEMOCRATIC SOCIALIST', 'NATIONAL STATESMAN',
       'UNITED STATES PARTY', 'PUBLIC INTEREST INDEPENDENT', 'WORKERS',
       'SOCIALIST', 'NATIONAL DEMOCRATIC PARTY OF ALABAMA', 'STATESMAN',
       'PEOPLE BEFORE PROFITS', 'UNAFFILIATED-AMERICAN', 'CITIZENS',
       'WORKERS WORLD', 'BY PETITION', 'NO PARTY', 'FREE LIBERTARIAN',
       'RIGHT TO LIFE', 'WORKERS LEAGUE', 'NEW UNION',
       'NOMINATED BY PETITION', 'REPEAL TF807 (HR4277-79)', 'GRASSROOTS',
       'GOD, FAMILY, AND COUNTRY', 'CONSUMER', 'LABOR AND FARM',
       'CONSTITUTIONALIST', 'TISCH INDEPENDENT CITIZENS',
       'PROLIFE FISCAL CONSERVATIVE', 'CONTEMPT OF COURT',
       'UNAFFILIATED AMERICAN', 'ILLINOIS SOLIDARITY', 'NEW ALLIANCE',
       'THIRD PARTY', 'WORKERS AGAINST CONCESSIONS', 'PROGRESSIVE ISSUES',
       'DEMOCRATIC-FARMER-LABOR', 'INDEPENDENT PROGRESSIVE LINE',
       'POPULIST', 'CONCERNS OF PEOPLE', 'GREEN', 'CHSRITIAN PRO-LIFE',
       "PEROT'S INDEPENDENTS", 'A CONNECTICUT PARTY',
       'CONCERNED CITIZENS', 'NATURAL LAW', 'FREEDOM FOR LAROUCHE',
       'UNENROLLED', 'LAROUCHE WAS RIGHT', 'INDEPENDENCE',
       'PROPERTY RIGHTS', 'KEEP AMERICA FIRST', 'DAMN DRUG DEALERS',
       'INDEPENDENCE FUSION', 'PATRIOT', 'REFORM', 'U.S. TAXPAYERS',
       'SOCIALIST EQUALITY', 'RESOURCE', 'U.S. PACIFIST',
       'AMERICAN CONSTITUTION', 'TERM LIMITS', 'OTHER', 'TAXPAYERS',
       'MARIJUANA REFORM', 'PACIFIC', 'CONSTITUTIONAL',
       'VERMONT GRASSROOTS', 'REPUBLICAN (NOT IDENTIFIED ON BALLOT)',
       'NOT IDENTIFIED', 'DEMOCRAT (NOT IDENTIFIED ON BALLOT)',
       'TIMESIZING NOT DOWNSIZING', 'CITIZENS FIRST', 'WORKING FAMILIES',
       'DEMOCRATIC-NONPARTISAN LEAGUE', 'ALASKAN INDEPENDENCE',
       'INDEPENDENT PARTY OF DELAWARE', 'ALASKA LIBERTARIAN',
       'NONPARTISAN', 'UNAFFILIATED', 'VETERANS PARTY OF AMERICA',
       'BUILDERS', 'PACIFIC GREEN', 'UNITED CITIZENS', 'VERMONT GREEN',
       'MARIJUANA', 'CONNECTICUT FOR LIEBERMAN', 'PROGRESSIVE',
       'POOR PEOPLEﾇ心 CAMPAIGN', 'GOD WE TRUST',
       'LEGALIZE MARIJUANA (G.R.I.P.)', 'SOLIDARITY',
       'SOCIALIST PARTY USA', 'PERSONAL CHOICE', 'DESERT GREENS',
       'ANTI-BUSHIST CANDIDATE', 'INDEPENDENT GREEN', 'WISCONSIN GREEN',
       'NO PARTY AFFILIATION', 'AMERICAN CONSTITUTION PARTY',
       'NEBRASKA PARTY', 'BOSS FOR SENATE', "POOR PEOPLE'S CAMPAIGN",
       'NONAFFILIATED', 'INDEPENDENT REFORM',
       'CONSTITUTION PARTY OF FLORIDA', 'TEA PARTY OF NEVADA',
       'RENT IS 2 DAMN HIGH', 'TAX REVOLT', 'ANTI-PROHIBITION',
       'UNITED STATES MARIJUANA', 'MOUNTAIN PARTY',
       'CONSTITUTION PARTY OF WISCONSIN', 'INDEPENDENT FOR MAINE',
       'INDEPENDENT FOR LIBERTY', 'GRASSROOTS PARTY',
       'MINNESOTA OPEN PROGRESSIVES', 'RESPONSIBILITY FAIRNESS INTEGRITY',
       'AMERICA FIRST', 'JERSEY STRONG INDEPENDENTS',
       'TOTALLY INDEPENDENT CANDIDATE', 'NO SLOGAN', 'REFORM NATION',
       'COMMON SENSE', 'JUSTICE', 'VOTEKISS', 'PEACE AND PROSPERITY',
       'I.D.E.A.', 'COUNTRY PARTY', 'UNITY PARTY OF COLORADO',
       'BOB QUAST FOR TERM LIMITS', 'D-R PARTY', 'ECONOMIC GROWTH',
       'PETITION', 'MOUNTAIN', 'AMERICAN SHOPPING PARTY',
       'NEW INDEPENDENT PARTY IOWA', "WOMEN'S EQUALITY", 'US TAXPAYERS',
       'LEGAL MARIJUANA NOW', 'MINNESOTA GREEN', 'FOR THE PEOPLE',
       'NEW DAY NJ', 'MAKE IT SIMPLE', 'DEMOCRATIC-NPL',
       'ALASKAN INDEPENDENCE PARTY', 'INDEPENDENT REPUBLICAN PARTY',
       'UNITY PARTY', 'NON-AFFILIATED', 'APPROVAL VOTING PARTY',
       'CONSTITUTION PARTY', 'WILLIE WILSON PARTY', 'OF, BY, FOR!'],
      dtype=object)


to_drop = senate_data[senate_data['party_detailed'] != 'DEMOCRAT'].index & senate_data[senate_data['party_detailed'] != 'REPUBLICAN'].index
senate_data.drop(to_drop,inplace=True)
senate_data.party_detailed.unique()

array(['REPUBLICAN', 'DEMOCRAT'], dtype=object)


senate_data.state.unique()

array(['Arizona', 'California', 'Connecticut', 'Delaware', 'Florida',
       'Hawaii', 'Indiana', 'Maine', 'Maryland', 'Massachusetts',
       'Michigan', 'Minnesota', 'Mississippi', 'Missouri', 'Montana',
       'Nebraska', 'Nevada', 'New Jersey', 'New Mexico', 'New York',
       'North Dakota', 'Ohio', 'Pennsylvania', 'Rhode Island',
       'Tennessee', 'Texas', 'Utah', 'Vermont', 'Virginia', 'Washington',
       'West Virginia', 'Wisconsin', 'Wyoming', 'Alabama', 'Alaska',
       'Arkansas', 'Colorado', 'Georgia', 'Idaho', 'Illinois', 'Iowa',
       'Kansas', 'Kentucky', 'Louisiana', 'New Hampshire',
       'North Carolina', 'Oklahoma', 'Oregon', 'South Carolina',
       'South Dakota'], dtype=object)


has_election = ['Alabama', 'Alaska', 'Arizona', 'Arkansas', 'California', 'Connecticut', 
'Florida', 'Georgia', 'Hawaii', 'Idaho', 'Illinois', 'Indiana', 'Iowa', 'Kansas', 
'Kentucky', 'Louisiana', 'Maryland', 'Missouri', 'Nevada', 'New Hampshire', 
'New York', 'North Carolina', 'North Dakota', 'Ohio', 'Oklahoma', 
'Oregon', 'Pennsylvania', 'South Carolina', 'South Dakota', 'Utah', 
'Vermont', 'Washington','Wisconsin']


no_elections = []
for i in senate_data.state.unique():
    if i not in has_election:
        print(i)
        no_elections.append(i)

Delaware
Maine
Massachusetts
Michigan
Minnesota
Mississippi
Montana
Nebraska
New Jersey
New Mexico
Rhode Island
Tennessee
Texas
Virginia
West Virginia
Wyoming
Colorado


senate_data = senate_data.loc[~senate_data['state'].isin(no_elections)]
print(senate_data.shape)
senate_data.state.unique()

(1207, 19)

array(['Arizona', 'California', 'Connecticut', 'Florida', 'Hawaii',
       'Indiana', 'Maryland', 'Missouri', 'Nevada', 'New York',
       'North Dakota', 'Ohio', 'Pennsylvania', 'Utah', 'Vermont',
       'Washington', 'Wisconsin', 'Alabama', 'Alaska', 'Arkansas',
       'Georgia', 'Idaho', 'Illinois', 'Iowa', 'Kansas', 'Kentucky',
       'Louisiana', 'New Hampshire', 'North Carolina', 'Oklahoma',
       'Oregon', 'South Carolina', 'South Dakota'], dtype=object)


## adding sum_per_party columns, which sums the votes that each party got in each state 
senate_data['sum_per_party'] = senate_data.groupby(['state','party_detailed','year'])['candidatevotes'].transform(sum)
senate_data


## for each state, the highest sum of votes will be kept 
senate_grouped = senate_data.loc[senate_data.groupby(['state','year'])['sum_per_party'].idxmax()]
senate_grouped.reset_index(inplace=True,drop=True)


for i,r in senate_grouped.iterrows():
    label = r.party_detailed
    senate_grouped.loc[i,'label'] = label


senate_grouped


senate_grouped.replace(['DEMOCRAT','REPUBLICAN'],[1,0],inplace=True)
senate_grouped.reset_index(inplace=True,drop=True)
senate_grouped


## we will drop features that has only one value
to_drop = []
for i in senate_grouped.columns:
    if len(senate_grouped[i].unique()) < 2:
        to_drop.append(i)
to_drop

['office', 'district', 'mode', 'version']


senate_grouped.drop(to_drop,inplace=True,axis=1)
senate_grouped.shape

(501, 17)


senate_grouped.columns

Index(['year', 'state', 'state_po', 'state_fips', 'state_cen', 'state_ic',
       'stage', 'special', 'candidate', 'party_detailed', 'writein',
       'candidatevotes', 'totalvotes', 'unofficial', 'party_simplified',
       'sum_per_party', 'label'],
      dtype='object')


state_po = senate_grouped['state_po'].unique()
state_po

array(['AL', 'AK', 'AZ', 'AR', 'CA', 'CT', 'FL', 'GA', 'HI', 'ID', 'IL',
       'IN', 'IA', 'KS', 'KY', 'LA', 'MD', 'MO', 'NV', 'NH', 'NY', 'NC',
       'ND', 'OH', 'OK', 'OR', 'PA', 'SC', 'SD', 'UT', 'VT', 'WA', 'WI'],
      dtype=object)


senate_grouped.drop(['state_po','state_cen','state_ic','candidate','party_detailed','candidatevotes','party_simplified','sum_per_party'],inplace=True,axis=1)
senate_grouped


fig = plt.figure(figsize=(10, 8))
fig.subplots_adjust(hspace=0.4, wspace=0.3)
colors = plt.get_cmap('tab10').colors

for idx, ft, color in zip(range(3),['writein','stage','special','unofficial'], colors):
    ax = fig.add_subplot(2, 2, idx+1)
    senate_grouped[ft].value_counts().plot(kind='barh',color = color)
    plt.title('True vs False values in '+ ft+' feature')
    plt.xlabel('Count')
    plt.show()


senate_grouped.drop(['writein','stage','special','unofficial'],inplace=True,axis=1)
senate_grouped.shape

(501, 5)


senate_grouped.head()


senate_grouped.year.unique()

array([1978, 1980, 1984, 1986, 1990, 1992, 1996, 1998, 2002, 2004, 2008,
       2010, 2014, 2016, 2020, 1976, 1982, 1988, 1994, 2000, 2006, 2012,
       2018, 2021], dtype=int64)


so_ec_fts = ['bachelors_2019','bachelors_2010','unemployed_2010','unemployed_2017',
'poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019','per_capita_income_2010',
'per_capita_income_2019']
pops = ['pop2010','pop2017','pop_2019']
democ1 = county_complete.loc[:,['state','name']+pops+['white_2010','white_2019','black_2010','black_2019','hispanic_2010'
                                ,'hispanic_2019']+so_ec_fts]
democ1.isnull().sum()

state                      0
name                       0
pop2010                    0
pop2017                    3
pop_2019                   0
white_2010                 0
white_2019                 0
black_2010                17
black_2019                 0
hispanic_2010              0
hispanic_2019              0
bachelors_2019             0
bachelors_2010             0
unemployed_2010            3
unemployed_2017            3
poverty_2017               2
poverty_2010               0
hs_grad_2010               0
hs_grad_2019               0
per_capita_income_2010     0
per_capita_income_2019     0
dtype: int64


imputer = KNNImputer(n_neighbors=5, weights="distance")
imputed = imputer.fit_transform(democ1.drop(['state','name'],axis=1))

# replace values in the dataset with new imputed values
democ1[democ1.columns.drop(['name','state'])] = imputed

democ1.rename(columns={'pop_2019':'pop2019'},inplace=True)
democ1.head()


##converting these features from percentages to numeric population numbers

perc_toNumeric = ['bachelors_2019','bachelors_2010','poverty_2017','poverty_2010','hs_grad_2010','hs_grad_2019',
               'white_2010','white_2019','black_2010','black_2019','hispanic_2010','hispanic_2019']

for i in perc_toNumeric:
    index = i.split('_')
    democ1[i] = democ1.apply(lambda row : int((row[i]/100)*row['pop'+index[-1]]),axis=1)

democ1.head()


## group by state and summing the features
democ_grouped = democ1.groupby(['state']).sum()
democ_grouped.reset_index(inplace=True)
democ_grouped.head()


## finding features data in missing years

all_fts = ['white','black','hispanic','bachelors', 'unemployed', 'poverty', 'hs_grad', 'per_capita_income']
for i in (range(2000,2024,2)): ## change 2022 -> 2024
    for j in all_fts:
        ft = j+'_'
        if(ft +str(i) not in democ_grouped.columns):
            if ft+'2010' in cols:
                pop1 = ft+'2010'
                y1 = 2010
                if ft+'2017' in cols:
                    pop2 = ft+'2017'
                    y2=2017
                else:
                    pop2 = ft+'2019'
                    y2 = 2019
            democ_grouped[ft+str(i)] = democ_grouped.apply(lambda row : (estimated_pop_based_on_gr(i,row[pop1],row[pop2],y1,y2)),axis=1)

            

                    
for i in all_fts:
    ft = i+'_'
    if(ft +str(2021) not in democ_grouped.columns):
            if ft+'2010' in cols:
                pop1 = ft+'2010'
                y1 = 2010
                if ft+'2017' in cols:
                    pop2 = ft+'2017'
                    y2=2017
                else:
                    pop2 = ft+'2019'
                    y2 = 2019
            democ_grouped[ft+str(2021)] = democ_grouped.apply(lambda row : (estimated_pop_based_on_gr(2021,row[pop1],row[pop2],y1,y2)),axis=1)
            

democ_grouped.head()


## dropping odd years because they are not included in elections
democ_grouped.drop(['pop2010','pop2017','pop2019','white_2019','black_2019','hispanic_2019',
                   'bachelors_2019','unemployed_2017','poverty_2017','hs_grad_2019','per_capita_income_2019',
                   ],inplace=True,axis=1)


final_e = pd.DataFrame(columns=['year','state']+all_fts)
row_data = 0
index=0
for i in range(2000,2024,2):
    row_data=0
    new_row = []
    new_row.append(i)
    for j in democ_grouped.state.unique():
        new_row.append(j)
        for ft in all_fts:
            val= democ_grouped.loc[row_data,[ft+"_"+str(i)]]
            new_row.append(val[0])            
        final_e.loc[index] = new_row
        new_row = []
        new_row.append(i)
        index+=1
        row_data+=1
final_e


row_data = 0
new_row = []
new_row.append(str(2021))
new_row.append('Georgia')
for ft in all_fts:
    val= democ_grouped.loc[row_data,[ft+"_"+str(2021)]]
    new_row.append(val[0])            

final_e.loc[index] = new_row

            

final_e


senate_grouped = senate_grouped[senate_grouped.year>=2000]
senate_grouped


xe_test = final_e.loc[final_e['state'].isin(has_election)]
xe_test = xe_test.loc[xe_test.year == 2022]
xe_test.head()


final_e = pd.merge(final_e,senate_grouped,on=['state','year'])
final_e


final_e.year.unique()

array([2000, 2002, 2004, 2006, 2008, 2010, 2012, 2014, 2016, 2018, 2020],
      dtype=object)


final_e.drop(['state_fips','totalvotes'],inplace=True,axis=1)
final_e


xe_train = final_e.drop(['state','label'],axis=1)
ye_train = final_e['label']


%%time
##scaling for svc

scaler = StandardScaler()
xe_train1 = scaler.fit_transform(xe_train) 
xe_test1 = scaler.fit_transform(xe_test.drop('state',axis=1)) 

svc=SVC()
svc_params = [{'kernel':['linear'],'C': [0.001, 0.01,0.05,0.1]},
             {'gamma': [0.01,0.02,0.03,0.04,0.05,1,5,9]}]
model_svc = GridSearchCV(svc, svc_params,cv=5,scoring='accuracy',verbose=3,n_jobs=-1)
model_svc.fit(xe_train1, ye_train)

print(model_svc.best_params_)
xe_train1.shape

Fitting 5 folds for each of 12 candidates, totalling 60 fits
{'gamma': 0.03}
Wall time: 106 ms

(234, 9)


y_pred_svc=model_svc.predict(xe_test1)


def senate_graph(df,algo):
    
    df['text'] = df.apply(lambda x: x['state'] +'<br>' + 'Prediction: ' + format(x['label']),axis=1)
    condition = [df['label'] == 'Democrat', df['label'] == 'Republican']
    choices = ['D','R','N']
    fig = px.choropleth(locations=state_po,color = df['label'],
                                       color_discrete_map = {'Democrat':'#FF0000','Republican':'#0015BC'},
                                       locationmode = 'USA-states',hover_name=df['state']
                                       )
                                           
                                           
    fig.update_layout(
    title_text = 'Predictions for 2022 Senate Elections - '+algo,
    geo_scope='usa'
    )
    fig.show()


y_pred_svc = y_pred_svc.ravel()
to_plot = xe_test.copy()
to_plot = to_plot.assign(label = y_pred_svc)
to_plot.replace([1,0],['Democrat','Republican'],inplace=True)
senate_graph(to_plot,'SVC')
to_plot.loc[:,['state','label']]


%%time
ab_clf = AdaBoostClassifier()

ab_params = {
    'n_estimators': [50, 100, 200, 250, 300],
    'learning_rate'  : [0.1, 0.3, 0.5, 0.7, 1.0]
}

ab_cv = GridSearchCV(ab_clf, ab_params, cv=5, verbose=1, n_jobs=-1)
ab_cv.fit(xe_train, ye_train)

print("Best: %f using %s" % (ab_cv.best_score_, ab_cv.best_params_))
y_pred_ab = ab_cv.predict(xe_test.drop('state',axis=1))

Fitting 5 folds for each of 25 candidates, totalling 125 fits
Best: 0.718039 using {'learning_rate': 0.7, 'n_estimators': 50}
Wall time: 11.2 s


y_pred_ab = y_pred_ab.ravel()
to_plot2 = xe_test.copy()
to_plot2 = to_plot2.assign(label = y_pred_ab)
to_plot2.replace([1,0],['Democrat','Republican'],inplace=True)
senate_graph(to_plot2,'AdaBoost')
to_plot2.loc[:,['state','label']]


%%time
rf_1=RandomForestClassifier()

# Create the param grid
rf_params = {'n_estimators': [10,50,100,200,300],
               'criterion': ['gini'], 
               'max_depth': [4,5,6,7,8,9,10],
               'min_samples_leaf': [1,3,5]}



rf_cv = GridSearchCV(
    rf_1, rf_params,
    cv=5, n_jobs=-1,
)
rf_cv.fit(xe_train, ye_train)

print('Best accuracy: %.3f' % rf_cv.best_score_)
print('\nBest params:\n', rf_cv.best_params_)

Best accuracy: 0.723

Best params:
 {'criterion': 'gini', 'max_depth': 8, 'min_samples_leaf': 1, 'n_estimators': 100}
Wall time: 31.5 s


y_pred_rf = rf_cv.predict(xe_test.drop('state',axis=1))

y_pred_rf = y_pred_rf.ravel()
to_plot3 = xe_test.copy()
to_plot3 = to_plot3.assign(label = y_pred_ab)
to_plot3.replace([1,0],['Democrat','Republican'],inplace=True)
senate_graph(to_plot3,'Random Forrest')
to_plot3.loc[:,['state','label']]


%%time
knn_clf = KNeighborsClassifier()

knn_clf_params = { 'n_neighbors' : [1,3,5,7,9,11,13,15],
               'weights' : ['uniform','distance'],
               'metric' : ['minkowski','euclidean','manhattan']}

knn_clf_cv = GridSearchCV(knn_clf, knn_clf_params, verbose = 1, cv=5, n_jobs = -1)

knn_clf_res = knn_clf_cv.fit(xe_train1, ye_train)
knn_clf_res.best_params_

y_pred_knn_clf = knn_clf_res.best_estimator_.predict(xe_test1)

Fitting 5 folds for each of 48 candidates, totalling 240 fits
Wall time: 285 ms


y_pred_knn_clf = y_pred_knn_clf.ravel()
to_plot4 = xe_test.copy()
to_plot4 = to_plot4.assign(label = y_pred_knn_clf)
to_plot4.replace([1,0],['Democrat','Republican'],inplace=True)
senate_graph(to_plot4,'KNN')
to_plot4.loc[:,['state','label']]


senate_graph(to_plot,'SVC')
senate_graph(to_plot2,'AdaBoost')
senate_graph(to_plot3,'Random Forrest')
senate_graph(to_plot4,'KNN')

	FIPS	State	County	Land Area
0	1001	Alabama	Autauga	1543.7
1	1003	Alabama	Baldwin	4135.0
2	1005	Alabama	Barbour	2292.1
3	1007	Alabama	Bibb	1611.9
4	1009	Alabama	Blount	1672.3

	FIPS	State	County	Land Area
157	6001	California	Alameda	1910.1
158	6003	California	Alpine	1913.1
159	6005	California	Amador	1534.7
160	6007	California	Butte	4246.6
161	6009	California	Calaveras	2642.3
...	...	...	...	...
3106	56037	Wyoming	Sweetwater	27003.0
3107	56039	Wyoming	Teton	10380.6
3108	56041	Wyoming	Uinta	5391.7
3109	56043	Wyoming	Washakie	5802.0
3110	56045	Wyoming	Weston	6210.6

	fips	state	name	pop2000	pop2010	pop2011	pop2012	pop2013	pop2014	pop2015	...	poverty_under_18_2019	two_plus_races_2019	unemployment_rate_2019	uninsured_2019	uninsured_65_and_older_2019	uninsured_under_19_2019	uninsured_under_6_2019	veterans_2019	white_2019	white_not_hispanic_2019
0	1001	Alabama	Autauga County	43671.0	54571	55199.0	54927.0	54695.0	54864.0	54838.0	...	23.2	2.2	3.5	7.1	0.0	1.7	1.7	12.6	76.8	74.6
1	1003	Alabama	Baldwin County	140415.0	182265	186534.0	190048.0	194736.0	199064.0	202863.0	...	13.4	1.7	4.0	8.9	0.3	3.8	2.2	11.8	86.2	83.1
2	1005	Alabama	Barbour County	29038.0	27457	27351.0	27175.0	26947.0	26749.0	26264.0	...	50.1	1.2	9.4	11.3	0.3	3.3	3.4	6.6	46.8	45.8
3	1007	Alabama	Bibb County	20826.0	22915	22745.0	22658.0	22503.0	22533.0	22561.0	...	NaN	0.6	7.0	10.7	0.0	2.0	4.5	8.0	76.8	74.5
4	1009	Alabama	Blount County	51024.0	57322	57562.0	57595.0	57623.0	57546.0	57590.0	...	18.4	1.6	3.1	10.8	0.2	5.9	6.1	7.7	95.5	86.9

	fips	State	County	Year	Male life expectancy (years)	Years behind international frontier (male)	Female life expectancy (years)	Years behind international frontier (female)	White male life expectancy (years)	White female life expectancy (years)	...	Closest ranked countries for male life expectancy (lower)	Closest ranked countries for female life expectancy (lower)	Rank (male)	Rank (female)	Male life expectancy change 1987 to 2007 (years)	Female life expectancy change 1987 to 2007 (years)	Male life expectancy change 1987 to 1997 (years)	Female life expectancy change 1987 to 1997 (years)	Male life expectancy change 1997 to 2007 (years)	Female life expectancy change 1997 to 2007 (years)
3927	6001	CALIFORNIA	ALAMEDA	1987	71.6	12	78.1	10	72.1	78.6	...	Barbados,Germany,Ireland,Netherlands Antilles,...	Austria,Denmark,Germany,United Kingdom,Virgin ...	1444	2262	6.1	4.2	2.6	1.9	3.5	2.3
3928	6001	CALIFORNIA	ALAMEDA	1988	71.6	12	78.0	11	72.2	78.6	...	Aruba,Austria,Netherlands Antilles,Singapore,U...	Costa Rica,Denmark,Montenegro,New Zealand,Virg...	1538	2310	6.1	4.2	2.6	1.9	3.5	2.3
3929	6001	CALIFORNIA	ALAMEDA	1989	71.5	14	78.2	11	72.1	78.8	...	Aruba,Finland,Guadeloupe,Luxembourg,Virgin Isl...	Costa Rica,Denmark,Guadeloupe,Montenegro,New Z...	1709	2212	6.1	4.2	2.6	1.9	3.5	2.3
3930	6001	CALIFORNIA	ALAMEDA	1990	71.9	13	78.4	12	72.5	79.0	...	Finland,Guadeloupe,Luxembourg,Netherlands Anti...	Costa Rica,Israel,Montenegro,New Zealand,Portugal	1547	2186	6.1	4.2	2.6	1.9	3.5	2.3
3931	6001	CALIFORNIA	ALAMEDA	1991	72.1	13	78.7	11	72.7	79.2	...	Barbados,Guadeloupe,Luxembourg,Netherlands Ant...	Costa Rica,Israel,Montenegro,New Zealand,Unite...	1515	2002	6.1	4.2	2.6	1.9	3.5	2.3
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
66082	56045	WYOMING	WESTON	2003	75.0	12	80.5	14	74.9	80.5	...	Brunei Darussalam,Chile,Finland,Korea, Republi...	Cuba,Denmark,Greece,Mayotte,Reunion	957	749	3.5	1.7	2.0	0.2	1.5	1.5
66083	56045	WYOMING	WESTON	2004	75.4	11	80.6	15	75.4	80.5	...	Chile,Denmark,Finland,Guadeloupe,Virgin Island...	Cuba,Denmark,Kuwait,Mayotte,Reunion	779	808	3.5	1.7	2.0	0.2	1.5	1.5
66084	56045	WYOMING	WESTON	2005	75.4	12	80.7	15	75.4	80.7	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Cuba,Denmark,French Guiana,Mayotte,Reunion	883	742	3.5	1.7	2.0	0.2	1.5	1.5
66085	56045	WYOMING	WESTON	2006	75.6	12	81.1	14	75.6	81.1	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Cuba,Denmark,Greece,Mayotte,Reunion	843	563	3.5	1.7	2.0	0.2	1.5	1.5
66086	56045	WYOMING	WESTON	2007	75.9	12	81.5	13	75.9	81.5	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Channel Islands,Greece,Malta,United Kingdom,Un...	727	393	3.5	1.7	2.0	0.2	1.5	1.5

	fips	State	County	Year	Male life expectancy (years)	Years behind international frontier (male)	Female life expectancy (years)	Years behind international frontier (female)	White male life expectancy (years)	White female life expectancy (years)	...	Closest ranked countries for male life expectancy (lower)	Closest ranked countries for female life expectancy (lower)	Rank (male)	Rank (female)	Male life expectancy change 1987 to 2007 (years)	Female life expectancy change 1987 to 2007 (years)	Male life expectancy change 1987 to 1997 (years)	Female life expectancy change 1987 to 1997 (years)	Male life expectancy change 1997 to 2007 (years)	Female life expectancy change 1997 to 2007 (years)
3927	6001	CALIFORNIA	ALAMEDA	1987	71.6	12	78.1	10	72.1	78.6	...	Barbados,Germany,Ireland,Netherlands Antilles,...	Austria,Denmark,Germany,United Kingdom,Virgin ...	1444	2262	6.1	4.2	2.6	1.9	3.5	2.3
3928	6001	CALIFORNIA	ALAMEDA	1988	71.6	12	78.0	11	72.2	78.6	...	Aruba,Austria,Netherlands Antilles,Singapore,U...	Costa Rica,Denmark,Montenegro,New Zealand,Virg...	1538	2310	6.1	4.2	2.6	1.9	3.5	2.3
3929	6001	CALIFORNIA	ALAMEDA	1989	71.5	14	78.2	11	72.1	78.8	...	Aruba,Finland,Guadeloupe,Luxembourg,Virgin Isl...	Costa Rica,Denmark,Guadeloupe,Montenegro,New Z...	1709	2212	6.1	4.2	2.6	1.9	3.5	2.3
3930	6001	CALIFORNIA	ALAMEDA	1990	71.9	13	78.4	12	72.5	79.0	...	Finland,Guadeloupe,Luxembourg,Netherlands Anti...	Costa Rica,Israel,Montenegro,New Zealand,Portugal	1547	2186	6.1	4.2	2.6	1.9	3.5	2.3
3931	6001	CALIFORNIA	ALAMEDA	1991	72.1	13	78.7	11	72.7	79.2	...	Barbados,Guadeloupe,Luxembourg,Netherlands Ant...	Costa Rica,Israel,Montenegro,New Zealand,Unite...	1515	2002	6.1	4.2	2.6	1.9	3.5	2.3
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
66082	56045	WYOMING	WESTON	2003	75.0	12	80.5	14	74.9	80.5	...	Brunei Darussalam,Chile,Finland,Korea, Republi...	Cuba,Denmark,Greece,Mayotte,Reunion	957	749	3.5	1.7	2.0	0.2	1.5	1.5
66083	56045	WYOMING	WESTON	2004	75.4	11	80.6	15	75.4	80.5	...	Chile,Denmark,Finland,Guadeloupe,Virgin Island...	Cuba,Denmark,Kuwait,Mayotte,Reunion	779	808	3.5	1.7	2.0	0.2	1.5	1.5
66084	56045	WYOMING	WESTON	2005	75.4	12	80.7	15	75.4	80.7	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Cuba,Denmark,French Guiana,Mayotte,Reunion	883	742	3.5	1.7	2.0	0.2	1.5	1.5
66085	56045	WYOMING	WESTON	2006	75.6	12	81.1	14	75.6	81.1	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Cuba,Denmark,Greece,Mayotte,Reunion	843	563	3.5	1.7	2.0	0.2	1.5	1.5
66086	56045	WYOMING	WESTON	2007	75.9	12	81.5	13	75.9	81.5	...	Brunei Darussalam,Chile,Korea, Republic of,Por...	Channel Islands,Greece,Malta,United Kingdom,Un...	727	393	3.5	1.7	2.0	0.2	1.5	1.5

Section B¶

Section C¶

Now we will be calaculating population who's age is over 18, which is number of people who are eligilble to vote¶

Creating Features¶

Now we are going to explore socio-economic features, such as education, employment and income¶

There is so many features, and we saw in an earlier section that features that are the same, but has different years, are highly correlated, so we will see how other different features , affect each other¶

Joining all features together for prediction part¶

Prediction¶

Gradient Boosting Regressor¶

Random Forrest Regressor¶

KNN¶

Predictions for California, Florida, South Dakota, Wyoming¶

Section D¶

Joining features together¶

Prediction¶

SVC¶

AdaBoost¶

Random Forrest Classsifier¶

KNN Classifier¶

Section E¶

Merging with labels, and extracting test set¶

SVC¶

Random Forrest Classifier¶

KNN Classifier¶

	fips	Year	Male life expectancy (years)	Years behind international frontier (male)	Female life expectancy (years)	Years behind international frontier (female)	White male life expectancy (years)	White female life expectancy (years)	Rank (male)	Rank (female)	Male life expectancy change 1987 to 2007 (years)	Female life expectancy change 1987 to 2007 (years)	Male life expectancy change 1987 to 1997 (years)	Female life expectancy change 1987 to 1997 (years)	Male life expectancy change 1997 to 2007 (years)	Female life expectancy change 1997 to 2007 (years)
0	6001	1997.0	74.371429	9.095238	80.000000	10.666667	74.590476	80.242857	805.428571	1123.380952	6.1	4.2	2.6	1.9	3.5	2.3
1	6003	1997.0	75.000000	6.095238	80.395238	8.619048	75.066667	80.347619	295.142857	560.428571	4.0	1.3	2.2	1.4	1.8	-0.1
2	6005	1997.0	74.790476	7.000000	80.214286	9.714286	75.023810	80.166667	389.142857	725.904762	4.0	1.7	2.7	0.6	1.3	1.1
3	6007	1997.0	73.276190	14.047619	79.538095	13.571429	73.233333	79.504762	1402.809524	1358.142857	2.0	0.9	1.3	0.2	0.7	0.7
4	6009	1997.0	74.323810	9.095238	80.495238	8.190476	74.304762	80.461905	711.904762	549.571429	4.5	2.4	1.8	1.7	2.7	0.7
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
209	56037	1997.0	73.995238	10.523810	79.952381	11.142857	74.000000	79.947619	882.238095	934.285714	2.2	1.3	1.2	1.1	1.0	0.2
210	56039	1997.0	76.700000	-1.047619	81.352381	3.666667	76.685714	81.352381	32.333333	141.285714	6.5	5.1	3.3	1.7	3.2	3.4
211	56041	1997.0	74.195238	9.714286	79.819048	12.000000	74.161905	79.814286	745.952381	1086.523810	3.0	1.5	1.2	0.4	1.8	1.1
212	56043	1997.0	73.947619	10.952381	79.819048	11.904762	73.919048	79.809524	916.238095	1073.047619	2.7	1.1	1.4	0.4	1.3	0.7
213	56045	1997.0	74.171429	9.809524	80.185714	9.904762	74.157143	80.176190	756.476190	764.238095	3.5	1.7	2.0	0.2	1.5	1.5

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	runoff	special	candidate	party	writein	mode	candidatevotes	totalvotes	unofficial	version	fusion_ticket
17025	2002	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	DICK COFFEE	LIBERTARIAN	False	TOTAL	2957	178687	False	20220331	False
17026	2002	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	JO BONNER	REPUBLICAN	False	TOTAL	108102	178687	False	20220331	False
17027	2002	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	JUDY MCCAIN BELK	DEMOCRAT	False	TOTAL	67507	178687	False	20220331	False
17028	2002	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	False	WRITEIN	NaN	True	TOTAL	121	178687	False	20220331	False
17029	2002	ALABAMA	AL	1	63	41	US HOUSE	2	GEN	False	False	CHARLES WOODS	DEMOCRAT	False	TOTAL	55495	187965	False	20220331	False
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
25475	2012	WYOMING	WY	56	83	68	US HOUSE	0	GEN	NaN	False	DANIEL CLYDE CUMMINGS	CONSTITUTION	False	TOTAL	4963	250700	False	20220331	False
25476	2012	WYOMING	WY	56	83	68	US HOUSE	0	GEN	NaN	False	DON WILLS	COUNTRY PARTY	False	TOTAL	3775	250700	False	20220331	False
25477	2012	WYOMING	WY	56	83	68	US HOUSE	0	GEN	NaN	False	OVER VOTE	NaN	False	TOTAL	600	250700	False	20220331	False
25478	2012	WYOMING	WY	56	83	68	US HOUSE	0	GEN	NaN	False	RICHARD P BRUBAKER	LIBERTARIAN	False	TOTAL	8442	250700	False	20220331	False
25479	2012	WYOMING	WY	56	83	68	US HOUSE	0	GEN	NaN	False	WRITEIN	NaN	True	TOTAL	416	250700	False	20220331	False

	year	state	state_fips	totalvotes_state/year
0	2002	ALABAMA	1	1268802
1	2002	ALASKA	2	227725
2	2002	ARIZONA	4	1194400
3	2002	ARKANSAS	5	688276
4	2002	CALIFORNIA	6	7258417
...	...	...	...	...
295	2012	VIRGINIA	51	3740455
296	2012	WASHINGTON	53	3006266
297	2012	WEST VIRGINIA	54	641354
298	2012	WISCONSIN	55	2866050
299	2012	WYOMING	56	250700

	pop2000	pop2010	pop2011	pop2012
state
Alabama	4447100.0	4779736	4798649.0	4813946.0
Alaska	626932.0	710231	716786.0	725255.0
Arizona	5130632.0	6392017	6465488.0	6544211.0
Arkansas	2673400.0	2915918	2938640.0	2949208.0
California	33871648.0	37253956	37672654.0	38019006.0
Colorado	4301261.0	5029196	5116411.0	5186330.0
Connecticut	3405565.0	3574097	3591927.0	3597705.0
Delaware	783600.0	897934	907884.0	916868.0
District of Columbia	572059.0	601723	620336.0	635630.0
Florida	15982378.0	18801310	19097369.0	19341327.0
Georgia	8186453.0	9687653	9810595.0	9911171.0
Hawaii	1211537.0	1360301	1378323.0	1392772.0
Idaho	1293953.0	1567582	1583180.0	1594673.0
Illinois	12419293.0	12830632	12862298.0	12878494.0
Indiana	6080485.0	6483802	6515358.0	6535665.0
Iowa	2926324.0	3046355	3063690.0	3074386.0
Kansas	2688418.0	2853118	2868756.0	2885316.0
Kentucky	4041769.0	4339367	4368505.0	4383673.0
Louisiana	4468976.0	4533372	4574388.0	4602681.0
Maine	1274923.0	1328361	1327968.0	1328101.0
Maryland	5296486.0	5773552	5843115.0	5891680.0
Massachusetts	6349097.0	6547629	6612178.0	6659627.0
Michigan	9938444.0	9883640	9876199.0	9886610.0
Minnesota	4919479.0	5303925	5345967.0	5377695.0
Mississippi	2844658.0	2967297	2977452.0	2982963.0
Missouri	5595211.0	5988927	6010280.0	6023267.0
Montana	902195.0	989415	996866.0	1003522.0
Nebraska	1711263.0	1826341	1841641.0	1854862.0
Nevada	1998257.0	2700551	2718170.0	2752410.0
New Hampshire	1235786.0	1316470	1318345.0	1320923.0
New Jersey	8414350.0	8791894	8844694.0	8882095.0
New Mexico	1819046.0	2059179	2077744.0	2083590.0
New York	18976457.0	19378102	19526372.0	19625409.0
North Carolina	8049313.0	9535483	9662940.0	9755299.0
North Dakota	642200.0	672591	684830.0	701380.0
Ohio	11353140.0	11536504	11543332.0	11546969.0
Oklahoma	3450654.0	3751351	3785232.0	3815298.0
Oregon	3421399.0	3831074	3865845.0	3893920.0
Pennsylvania	12281054.0	12702379	12742811.0	12768034.0
Rhode Island	1048319.0	1052567	1052154.0	1052761.0
South Carolina	4012012.0	4625364	4672744.0	4719009.0
South Dakota	754844.0	814180	823338.0	832576.0
Tennessee	5689283.0	6346105	6396281.0	6450632.0
Texas	20851820.0	25145561	25644424.0	26078327.0
Utah	2233169.0	2763885	2815430.0	2854222.0
Vermont	608827.0	625741	626210.0	625606.0
Virginia	7067927.0	7994802	8107548.0	8188656.0
Washington	5894121.0	6724540	6819155.0	6890899.0
West Virginia	1808344.0	1852994	1854891.0	1855360.0
Wisconsin	5363675.0	5686986	5705812.0	5721075.0
Wyoming	493782.0	563626	567602.0	576608.0

	state	name	above18_2010	above18_2019
0	Alabama	Autauga County	39945	42199
1	Alabama	Baldwin County	140344	166645
2	Alabama	Barbour County	21443	20060
3	Alabama	Bibb County	17713	17859
4	Alabama	Blount County	43220	44299

	state	above18_2010	above18_2019
0	Alabama	3646981	3780169
1	Alaska	522751	552625
2	Arizona	4763130	5413832
3	Arkansas	2204313	2295361
4	California	27955194	30258616

	state	name	white_2010	white_2019	black_2010	black_2019	hispanic_2010	hispanic_2019	pop_2010	pop_2019	pop_white_2010	pop_black_2010	pop_hispanic_2010	pop_white_2019	pop_black_2019	pop_hispanic_2019
0	Alabama	Autauga County	78.5	76.8	17.7	19.0	2.4	2.8	54571.0	55380.0	42838	9659	1309	42531	10522	1550
1	Alabama	Baldwin County	85.7	86.2	9.4	9.3	4.4	4.6	182265.0	212830.0	156201	17132	8019	183459	19793	9790
2	Alabama	Barbour County	48.0	46.8	46.9	47.6	5.1	4.4	27457.0	25361.0	13179	12877	1400	11868	12071	1115
3	Alabama	Bibb County	75.8	76.8	22.0	22.3	1.8	2.6	22915.0	22493.0	17369	5041	412	17274	5015	584
4	Alabama	Blount County	92.6	95.5	1.3	1.6	8.1	9.3	57322.0	57681.0	53080	745	4643	55085	922	5364

	pop_2010	pop_2019	pop_white_2010	pop_black_2010	pop_hispanic_2010	pop_white_2019	pop_black_2019	pop_hispanic_2019
state
Alabama	4779736.0	4876250.0	3275153	1251165	185865	3319782	1299186	208777
Alaska	710231.0	737068.0	473642	23395	39384	476038	24292	51891
Arizona	6392017.0	7050299.0	4666450	258956	1896799	5442793	316944	2208642
Arkansas	2915918.0	2999370.0	2245221	450245	185998	2301078	459343	224219
California	37253956.0	39283497.0	21459611	2296383	14011485	23457883	2269284	15331411

	state	percent_pop_white_2002	percent_pop_black_2002	percent_pop_hispanic_2002	percent_pop_white_2004	percent_pop_black_2004	percent_pop_hispanic_2004	percent_pop_white_2006	percent_pop_black_2006	percent_pop_hispanic_2006	...	percent_pop_white_2012	percent_pop_black_2012	percent_pop_hispanic_2012	above18_2010	above18_2019	above18_2002	above18_2004	above18_2006	above18_2008	above18_2012
0	Alabama	71.70	26.83	3.74	70.87	26.65	3.77	70.06	26.49	3.81	...	68.24	26.20	3.96	3646981	3780169	3534749	3562479	3590426	3618593	3675591
1	Alaska	73.27	3.52	4.95	71.50	3.46	5.09	69.81	3.40	5.23	...	65.38	3.25	5.73	522751	552625	498297	504301	510377	516527	529049
2	Arizona	76.44	4.10	31.11	75.35	4.07	30.66	74.43	4.06	30.27	...	73.58	4.12	29.90	4763130	5413832	4283516	4398691	4516962	4638413	4891200
3	Arkansas	80.73	16.25	5.88	79.74	16.04	6.00	78.79	15.83	6.12	...	76.54	15.33	6.55	2204313	2295361	2128110	2146910	2165876	2185010	2223786
4	California	57.61	6.72	37.58	57.57	6.57	37.57	57.56	6.43	37.57	...	57.52	6.02	37.56	27955194	30258616	26133587	26577547	27029049	27488221	28430099

	state	bachelors_2010	unemployed_2010	poverty_2010	hs_grad_2010	per_capita_income_2010	bachelors_2002	unemployed_2002	poverty_2002	hs_grad_2002	...	bachelors_2008	unemployed_2008	poverty_2008	hs_grad_2008	per_capita_income_2008	bachelors_2012	unemployed_2012	poverty_2012	hs_grad_2012	per_capita_income_2012
0	Alabama	1048348	2.314920e+05	816734	3895733	1328086.0	910264	1436289	758243	3650255	...	1011977	365352	801701	3832857	1277693	1086025	146676	832048	3959639	1380465
1	Alaska	191165	2.829818e+04	67295	643735	778091.0	172372	31935	59752	611470	...	186283	29166	65324	635512	746454	196174	27455	69324	652063	811068
2	Arizona	1681405	3.202520e+05	975382	5426296	308673.0	1418304	1077823	797297	4896406	...	1611375	433766	927440	5288676	297978	1754478	236443	1025801	5567496	319751
3	Arkansas	564663	1.108420e+05	524650	2390880	1404481.0	477689	515916	504115	2228812	...	541537	162807	519439	2349290	1345315	588776	75463	529912	2433205	1466248
4	California	11054760	2.244333e+06	5107134	29977852	1567459.0	9603418	14217050	4332364	27884825	...	10672556	3560556	4901332	29440309	1499916	11450651	1414675	5321576	30525209	1638042

	state	pop_2010	pop_2019	pop_white_2010	pop_black_2010	pop_hispanic_2010	pop_white_2019	pop_black_2019	pop_hispanic_2019	pop_white_2002	...	pop_hispanic_2004	pop_white_2006	pop_black_2006	pop_hispanic_2006	pop_white_2008	pop_black_2008	pop_hispanic_2008	pop_white_2012	pop_black_2012	pop_hispanic_2012
0	Alabama	4779736.0	4876250.0	3275153	1251165	185865	3319782	1299186	208777	3236277	...	172828	3255657	1230820	177069	3265390	1240951	181413	3284944	1261462	190425
1	Alaska	710231.0	737068.0	473642	23395	39384	476038	24292	51891	471528	...	33608	472583	23014	35433	473112	23204	37356	474171	23587	41521
2	Arizona	6392017.0	7050299.0	4666450	258956	1896799	5442793	316944	2208642	4114858	...	1727664	4381983	238927	1782296	4521980	248740	1838656	4815534	269591	1956779
3	Arkansas	2915918.0	2999370.0	2245221	450245	185998	2301078	459343	224219	2197358	...	166194	2221160	446303	172549	2233158	448269	179147	2257348	452228	193110
4	California	37253956.0	39283497.0	21459611	2296383	14011485	23457883	2269284	15331411	19901761	...	13233544	20666012	2308611	13487936	21059073	2302489	13747218	21867766	2290293	14280831

	state	name	pop2010	pop2017	pop_2019	bachelors_2019	bachelors_2010	unemployed_2010	unemployed_2017	poverty_2017	poverty_2010	hs_grad_2010	hs_grad_2019	per_capita_income_2010	per_capita_income_2019
0	Alabama	Autauga County	54571	55504.0	55380	26.6	21.7	2282.0	1001.0	13.7	10.6	85.3	88.5	24568	29819
1	Alabama	Baldwin County	182265	212628.0	212830	31.9	26.8	8339.0	3652.0	11.8	12.2	87.6	90.8	26469	32626
2	Alabama	Barbour County	27457	25270.0	25361	11.6	13.5	1262.0	486.0	27.2	25.0	71.9	73.2	15875	18473
3	Alabama	Bibb County	22915	22668.0	22493	10.4	10.0	1020.0	373.0	15.2	12.6	74.5	79.1	19918	20778
4	Alabama	Blount County	57322	58013.0	57681	13.1	12.5	2446.0	985.0	15.6	13.4	74.7	80.5	21070	24747

	year	state	bachelors	unemployed	poverty	hs_grad	per_capita_income	white	black	hispanic
0	2002	Alabama	910264	1436289	758243	3650255	1137701	2534415	948373	132199
1	2002	Alaska	172372	31935	59752	611470	659056	365102	17540	24665
2	2002	Arizona	1418304	1077823	797297	4896406	268067	3274319	175624	1332601
3	2002	Arkansas	477689	515916	504115	2228812	1182358	1718023	345817	125132
4	2002	California	9603418	14217050	4332364	27884825	1314256	15055559	1756177	9821001
...	...	...	...	...	...	...	...	...	...	...
295	2012	Virginia	2831423	230261	858773	7036495	3542831	4213023	1192141	503168
296	2012	Washington	2170056	259563	840662	6158413	998635	4033491	189613	603074
297	2012	West Virginia	335165	56279	321913	1532898	1120512	1362031	52470	18415
298	2012	Wisconsin	1528961	160342	675328	5138991	1846759	3762747	277439	266043
299	2012	Wyoming	138528	16417	57869	520391	647999	386016	3712	38640

	year	state	bachelors	unemployed	poverty	hs_grad	per_capita_income	white	black	hispanic	voter_turnout
0	2002	Alabama	910264	1436289	758243	3650255	1137701	2534415	948373	132199	0.359
1	2002	Alaska	172372	31935	59752	611470	659056	365102	17540	24665	0.457
2	2002	Arizona	1418304	1077823	797297	4896406	268067	3274319	175624	1332601	0.279
3	2002	Arkansas	477689	515916	504115	2228812	1182358	1718023	345817	125132	0.323
4	2002	California	9603418	14217050	4332364	27884825	1314256	15055559	1756177	9821001	0.278

	Regressor	MSE
0	GradientBoostingRegressor	0.00811
0	RandomForestRegressor	0.00761
0	KNeighborsRegressor	0.00555

	state	year	voter_turnout	predicted_voter_turnout
0	California	2010	0.345	0.340667
1	Florida	2010	0.346	0.322000
2	South Dakota	2010	0.522	0.619333
3	Wyoming	2010	0.446	0.492667
4	California	2012	0.429	0.397333
5	Florida	2012	0.495	0.399333
6	South Dakota	2012	0.582	0.619333
7	Wyoming	2012	0.581	0.492667

	state	total_mse	mean_true_turnout	mean_pred_turnout
45	Nebraska	0.027585	0.457	0.516
46	Virginia	0.030333	0.478	0.484
47	Texas	0.030447	0.334	0.435
48	Tennessee	0.032172	0.393	0.499
49	Oklahoma	0.038457	0.373	0.478

	state	total_mse	mean_true_turnout	mean_pred_turnout
45	Minnesota	0.028287	0.605	0.523
46	Virginia	0.029658	0.478	0.487
47	Tennessee	0.036222	0.393	0.507
48	Oklahoma	0.036405	0.373	0.472
49	Texas	0.039108	0.334	0.453

	state	total_mse	mean_true_turnout	mean_pred_turnout
0	South Dakota	0.001875	0.552	0.546
1	Vermont	0.003591	0.530	0.524
2	Alaska	0.004718	0.518	0.480
3	Delaware	0.005945	0.496	0.497
4	Kentucky	0.006587	0.465	0.472
5	Kansas	0.007572	0.444	0.479
6	Connecticut	0.007740	0.471	0.449
7	South Carolina	0.008281	0.438	0.451
8	California	0.008362	0.387	0.442
9	Alabama	0.008499	0.451	0.459
10	Washington	0.008639	0.526	0.478
11	Rhode Island	0.008761	0.460	0.497
12	Wyoming	0.009159	0.513	0.509
13	Ohio	0.009364	0.507	0.503
14	North Dakota	0.009885	0.522	0.529
15	Nevada	0.010175	0.406	0.443
16	New Mexico	0.010430	0.440	0.489
17	Montana	0.010665	0.544	0.546
18	Pennsylvania	0.012348	0.479	0.503
19	Oregon	0.012701	0.523	0.455
20	Missouri	0.013028	0.501	0.507
21	Utah	0.013617	0.425	0.465
22	Michigan	0.014614	0.513	0.503
23	Illinois	0.014710	0.450	0.492
24	Louisiana	0.015757	0.399	0.434

	state	total_mse	mean_true_turnout	mean_pred_turnout
0	Arkansas	0.001845	0.409	0.405
1	Kentucky	0.004075	0.465	0.470
2	Washington	0.007651	0.526	0.552
3	Connecticut	0.008984	0.471	0.506
4	Arizona	0.009625	0.400	0.455
5	California	0.011359	0.387	0.449
6	Vermont	0.012953	0.530	0.601
7	South Carolina	0.013177	0.438	0.458
8	Nevada	0.013550	0.406	0.465
9	South Dakota	0.014540	0.552	0.633
10	Delaware	0.017815	0.496	0.570
11	Alaska	0.019067	0.518	0.610
12	Wisconsin	0.020403	0.573	0.535
13	Alabama	0.020841	0.451	0.424
14	Oregon	0.021319	0.523	0.625
15	Wyoming	0.023265	0.513	0.578
16	Iowa	0.023828	0.567	0.632
17	Maryland	0.024320	0.495	0.534
18	Utah	0.024832	0.425	0.511
19	Florida	0.025084	0.420	0.507
20	Texas	0.025543	0.334	0.424
21	West Virginia	0.025574	0.395	0.503
22	Minnesota	0.025780	0.605	0.668
23	Maine	0.027404	0.610	0.698
24	North Dakota	0.027456	0.522	0.621

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	special	candidate	party	writein	mode	candidatevotes	totalvotes	unofficial	version	fusion_ticket
0	1976	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	BILL DAVENPORT	DEMOCRAT	False	TOTAL	58906	157170	False	20220331	False
1	1976	ALABAMA	AL	1	63	41	US HOUSE	1	GEN	False	JACK EDWARDS	REPUBLICAN	False	TOTAL	98257	157170	False	20220331	False
3	1976	ALABAMA	AL	1	63	41	US HOUSE	2	GEN	False	J CAROLE KEAHEY	DEMOCRAT	False	TOTAL	66288	156362	False	20220331	False
4	1976	ALABAMA	AL	1	63	41	US HOUSE	2	GEN	False	WILLIAM L "BILL" DICKINSON	REPUBLICAN	False	TOTAL	90069	156362	False	20220331	False
6	1976	ALABAMA	AL	1	63	41	US HOUSE	3	GEN	False	BILL NICHOLS	DEMOCRAT	False	TOTAL	106935	108048	False	20220331	False

	state	pop2010	pop2017	pop2019	white_2010	white_2019	black_2010	black_2019	hispanic_2010	hispanic_2019	bachelors_2019	bachelors_2010	unemployed_2010	unemployed_2017	poverty_2017	poverty_2010	hs_grad_2010	hs_grad_2019	per_capita_income_2010	per_capita_income_2019
0	Alabama	4779736.0	4.874747e+06	4876250.0	3275153	3319782	1251165	1299186	185865	208777	1248481	1048348	2.314920e+05	95342.000000	873814	816734	3895733	4204871	1328086.0	1611293.0
1	Alaska	710231.0	7.408602e+05	737068.0	473642	476038	23395	24292	39384	51891	216536	191165	2.829817e+04	25608.760753	75172	67295	643735	683395	778091.0	959114.0
2	Arizona	6392017.0	7.016270e+06	7050299.0	4666450	5442793	258956	316944	1896799	2208642	2084788	1681405	3.202520e+05	161314.000000	1187525	975382	5426296	6140741	308673.0	367504.0
3	Arkansas	2915918.0	3.004279e+06	2999370.0	2245221	2301078	450245	459343	185998	224219	697251	564663	1.108420e+05	49835.000000	543691	524650	2390880	2596846	1404481.0	1746378.0
4	California	37253956.0	3.953665e+07	39283497.0	21459611	23457883	2296383	2269284	14011485	15331411	13156289	11054760	2.244333e+06	918883.000000	5976401	5107134	29977852	32648238	1567459.0	1960320.0

	year	state	white	black	hispanic	bachelors	unemployed	poverty	hs_grad	per_capita_income
0	2000	Alabama	1094533	1094533	1094533	878684	2266828	744287	3591341	1094533
1	2000	Alaska	632260	632260	632260	167970	32916	58003	603660	632260
2	2000	Arizona	258779	258779	258779	1359233	1459861	758108	4772226	258779
3	2000	Arkansas	1132550	1132550	1132550	458125	757788	499109	2190042	1132550
4	2000	California	1257624	1257624	1257624	9271392	22554855	4157783	27384813	1257624
...	...	...	...	...	...	...	...	...	...	...
556	2020	Virginia	4154049	4154049	4154049	3296255	82882	997684	7628024	4154049
557	2020	Washington	1202120	1202120	1202120	2628024	77814	939370	6783314	1202120
558	2020	West Virginia	1350557	1350557	1350557	380135	23226	322709	1585885	1350557
559	2020	Wisconsin	2222798	2222798	2222798	1757528	20816	735155	5360379	2222798
560	2020	Wyoming	741582	741582	741582	159739	8157	67904	543568	741582

	year	state	state_fips	totalvotes	label
12	2000	Alabama	1	165669	0
13	2002	Alabama	1	178687	0
14	2004	Alabama	1	255164	0
15	2006	Alabama	1	165841	0
16	2008	Alabama	1	214367	0
...	...	...	...	...	...
1128	2012	Wyoming	56	250700	0
1129	2014	Wyoming	56	171153	0
1130	2016	Wyoming	56	258788	0
1131	2018	Wyoming	56	201245	0
1132	2020	Wyoming	56	278503	0

	Classifier	Accuracy
0	SVC	76.470588
0	AdaBoostClassifier	85.300000
0	RandomForestClassifier	87.500000
0	KNeighborsClassifier	78.676000

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	special	candidate	party_detailed	writein	mode	candidatevotes	totalvotes	unofficial	version	party_simplified	sum_per_party
0	1976	Arizona	AZ	4	86	61	US SENATE	statewide	gen	False	SAM STEIGER	REPUBLICAN	False	total	321236	741210	False	20210114	REPUBLICAN	321236
1	1976	Arizona	AZ	4	86	61	US SENATE	statewide	gen	False	WM. MATHEWS FEIGHAN	DEMOCRAT	False	total	1565	741210	False	20210114	OTHER	412664
2	1976	Arizona	AZ	4	86	61	US SENATE	statewide	gen	False	DENNIS DECONCINI	DEMOCRAT	False	total	400334	741210	False	20210114	DEMOCRAT	412664
4	1976	Arizona	AZ	4	86	61	US SENATE	statewide	gen	False	BOB FIELD	DEMOCRAT	False	total	10765	741210	False	20210114	OTHER	412664
6	1976	California	CA	6	93	71	US SENATE	statewide	gen	False	S. I. (SAM) HAYAKAWA	REPUBLICAN	False	total	3748973	7470586	False	20210114	REPUBLICAN	3748973
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
3005	2020	South Dakota	SD	46	45	37	US SENATE	statewide	gen	False	DAN AHLERS	DEMOCRAT	False	total	143987	420219	True	20210114	DEMOCRAT	143987
3026	2021	Georgia	GA	13	58	44	US SENATE	statewide	runoff	True	KELLY LOEFFLER	REPUBLICAN	False	total	2194848	4483294	True	20210114	REPUBLICAN	4408827
3027	2021	Georgia	GA	13	58	44	US SENATE	statewide	runoff	True	RAPHAEL WARNOCK	DEMOCRAT	False	total	2288446	4483294	True	20210114	DEMOCRAT	4557708
3028	2021	Georgia	GA	13	58	44	US SENATE	statewide	runoff	False	DAVID A. PERDUE	REPUBLICAN	False	total	2213979	4483241	True	20210114	REPUBLICAN	4408827
3029	2021	Georgia	GA	13	58	44	US SENATE	statewide	runoff	False	JON OSSOFF	DEMOCRAT	False	total	2269262	4483241	True	20210114	DEMOCRAT	4557708

	year	state	state_po	state_fips	state_cen	state_ic	office	district	stage	special	...	party_detailed	writein	mode	candidatevotes	totalvotes	unofficial	version	party_simplified	sum_per_party	label
0	1978	Alabama	AL	1	63	41	US SENATE	statewide	gen	True	...	DEMOCRAT	False	total	401852	729842	False	20210114	DEMOCRAT	948906	DEMOCRAT
1	1980	Alabama	AL	1	63	41	US SENATE	statewide	gen	False	...	REPUBLICAN	False	total	650362	1296757	False	20210114	REPUBLICAN	650362	REPUBLICAN
2	1984	Alabama	AL	1	63	41	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	860535	1371234	False	20210114	DEMOCRAT	860535	DEMOCRAT
3	1986	Alabama	AL	1	63	41	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	609360	1211897	False	20210114	DEMOCRAT	609360	DEMOCRAT
4	1990	Alabama	AL	1	63	41	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	717814	1185154	False	20210114	DEMOCRAT	717814	DEMOCRAT
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
496	2006	Wisconsin	WI	55	35	25	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	25096	2138297	False	20210114	OTHER	1464310	DEMOCRAT
497	2010	Wisconsin	WI	55	35	25	US SENATE	statewide	gen	False	...	REPUBLICAN	False	total	134	2171331	False	20210114	REPUBLICAN	1126133	REPUBLICAN
498	2012	Wisconsin	WI	55	35	25	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	1547104	3009411	False	20210114	DEMOCRAT	1547104	DEMOCRAT
499	2016	Wisconsin	WI	55	35	25	US SENATE	statewide	gen	False	...	REPUBLICAN	False	total	1479471	2948741	False	20210114	REPUBLICAN	1479471	REPUBLICAN
500	2018	Wisconsin	WI	55	35	25	US SENATE	statewide	gen	False	...	DEMOCRAT	False	total	1472914	2657841	False	20210114	DEMOCRAT	1472914	DEMOCRAT

	year	state	state_fips	totalvotes	label
8	2002	Alabama	1	1353023	0
9	2004	Alabama	1	1839066	0
10	2008	Alabama	1	2060191	0
11	2010	Alabama	1	1485499	0
12	2014	Alabama	1	818090	0
...	...	...	...	...	...
496	2006	Wisconsin	55	2138297	1
497	2010	Wisconsin	55	2171331	0
498	2012	Wisconsin	55	3009411	1
499	2016	Wisconsin	55	2948741	0
500	2018	Wisconsin	55	2657841	1

	year	state	white	black	hispanic	bachelors	unemployed	poverty	hs_grad	per_capita_income
561	2022	Alabama	1675030	1675030	1675030	1295723	14978	913036	4295246	1675030
562	2022	Alaska	998140	998140	998140	223263	23603	80430	695351	998140
563	2022	Arizona	381400	381400	381400	2170333	51869	1319795	6330564	381400
564	2022	Arkansas	1818300	1818300	1818300	725696	11038	557029	2656342	1818300
565	2022	California	2041598	2041598	2041598	13653201	140768	6536657	33415609	2041598